diff --git a/apiconnector/src/test/java/apiconnector/TestDataFunctionality.java b/apiconnector/src/test/java/apiconnector/TestDataFunctionality.java deleted file mode 100644 index 7fcdb117..00000000 --- a/apiconnector/src/test/java/apiconnector/TestDataFunctionality.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * OpenmlApiConnector - Java integration of the OpenML Web API - * Copyright (C) 2014 - * @author Jan N. van Rijn (j.n.van.rijn@liacs.leidenuniv.nl) - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - */ -package apiconnector; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.net.URL; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathFactory; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.junit.Ignore; -import org.junit.Test; -import org.openml.apiconnector.algorithms.Conversion; -import org.openml.apiconnector.io.ApiException; -import org.openml.apiconnector.io.HttpConnector; -import org.openml.apiconnector.xml.Data; -import org.openml.apiconnector.xml.Data.DataSet; -import org.openml.apiconnector.xml.DataDelete; -import org.openml.apiconnector.xml.DataFeature; -import org.openml.apiconnector.xml.DataQuality; -import org.openml.apiconnector.xml.DataQualityList; -import org.openml.apiconnector.xml.DataReset; -import org.openml.apiconnector.xml.DataSetDescription; -import org.openml.apiconnector.xml.DataTag; -import org.openml.apiconnector.xml.DataUntag; -import org.openml.apiconnector.xml.TaskDelete; -import org.openml.apiconnector.xml.TaskInputs; -import org.openml.apiconnector.xml.TaskInputs.Input; -import org.openml.apiconnector.xml.TaskTag; -import org.openml.apiconnector.xml.TaskUntag; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; - -import testbase.BaseTestFramework; - -public class TestDataFunctionality extends BaseTestFramework { - public static final String data_file = "data/iris.arff"; - private static final int probe = 61; - private static final int probe_ignore_attribute = 44; - private static final String tag = "junittest"; - - @Ignore // while working on MinIo, the dataset layout might change slightly. hard to check - @Test - public void testApiDataDownload() throws Exception { - DataSetDescription dsd = client_read_test.dataGet(probe); - DataFeature features = client_read_test.dataFeatures(probe); - DataQuality qualities = client_read_test.dataQualities(probe, null); - - File tempDsd = Conversion.stringToTempFile(xstream.toXML(dsd), "data", "xml"); - File tempXsd = client_read_test.getXSD("openml.data.upload"); - - String url = client_read_test.getApiUrl() + "data/" + probe; - File rawFile = HttpConnector.getTempFileFromUrl(new URL(url + "?api_key=" + client_read_test.getApiKey()), "xml"); - String raw = Conversion.fileToString(rawFile); - - assertTrue(Conversion.validateXML(tempDsd, tempXsd)); - - - String dsdFromOpenml = toPrettyString(raw, 0); - String dsdFromConnector = toPrettyString(xstream.toXML(dsd), 0); - - if (!dsdFromOpenml.equals(dsdFromConnector)) { - System.out.println("===== OBTAINED FROM OPENML: ====="); - System.out.println(dsdFromOpenml); - System.out.println("===== LOCALLY AVILABLE: ====="); - System.out.println(dsdFromConnector); - } - - assertEquals(dsdFromOpenml, dsdFromConnector); - - // very easy checks, should all pass - assertTrue(dsd.getId() == probe); - assertTrue(features.getFeatures().length > 0); - assertTrue(qualities.getQualities().length > 0); - } - - @Test - public void testIgnoreAttributeNotEmpty() throws Exception { - DataSetDescription dsd = client_read_test.dataGet(probe_ignore_attribute); - assertTrue(dsd.getIgnore_attribute().length > 0); - } - - @Test - public void testApiDataUnprocessed() throws Exception { - client_admin_test.dataUnprocessed(2, "normal"); - client_admin_test.dataUnprocessed(2, "random"); - } - - @Test - public void testApiUploadDownload() throws Exception { - DataSetDescription dsd = new DataSetDescription("test", "Unit test should be deleted", "arff", "class"); - File toUpload = new File(data_file); - int dataId = client_write_test.dataUpload(dsd, toUpload); - client_admin_test.dataStatusUpdate(dataId, "active"); - DataReset dr = client_write_test.dataReset(dataId); - assertTrue(dr.get_id() == dataId); - DataTag dt = client_write_test.dataTag(dataId, tag); - assertTrue(Arrays.asList(dt.getTags()).contains(tag)); - - DataSetDescription dsd_downloaded = client_read_test.dataGet(dataId); - client_read_test.datasetGet(dsd_downloaded); - - // create task upon it (clustering task) - Input estimation_procedure = new Input("estimation_procedure", "17"); - Input data_set = new Input("source_data", "" + dataId); - Input measure = new Input("evaluation_measures", "predictive_accuracy"); - Input[] inputs = {estimation_procedure, data_set, measure}; - int taskId = client_write_test.taskUpload(new TaskInputs(null, 5, inputs, null)); - - TaskTag tt = client_write_test.taskTag(taskId, tag); - assertTrue(Arrays.asList(tt.getTags()).contains(tag)); - TaskUntag tu = client_write_test.taskUntag(taskId, tag); - assertNull(tu.getTags()); - - try { - client_write_test.dataDelete(dataId); - // this SHOULD fail, we should not be allowed to delete data that contains tasks. - fail("Problem with API. Dataset ("+dataId+") was deleted while it contains a task ("+taskId+"). "); - } catch(ApiException ae) {} - - - // delete the task - TaskDelete td = client_write_test.taskDelete(taskId); - assertEquals((Integer) taskId, td.get_id()); - - // and delete the data - DataUntag du = client_write_test.dataUntag(dataId, tag); - assertNull(du.getTags()); - - DataDelete dd = client_write_test.dataDelete(dataId); - assertEquals((Integer) dataId, dd.get_id()); - } - - @Test - public void testDataQualitiesWithNullValues() throws Exception { - DataQuality dq = client_read_live.dataQualities(3, null); - - // check if test is actually up to date (otherwise we should use other dataset that contains null values) - Collection qualityValues = dq.getQualitiesMap().values(); - assertTrue(qualityValues.contains(null)); - - // test if converting back doesn't break anything - xstream.toXML(dq); - } - - @Test - @Ignore - public void testApiUploadFromUrl() throws Exception { - String dataUrl = "http://storm.cis.fordham.edu/~gweiss/data-mining/weka-data/cpu.arff"; - - DataSetDescription dsd = new DataSetDescription("anneal", "Unit test should be deleted", "arff", dataUrl, "class"); - - int dataId = client_write_test.dataUpload(dsd, null); - DataTag dt = client_write_test.dataTag(dataId, tag); - assertTrue(Arrays.asList(dt.getTags()).contains(tag)); - - // Download dataset and check md5 thingy - DataSetDescription dsd_downloaded = client_read_test.dataGet(dataId); - client_read_test.datasetGet(dsd_downloaded); - - HttpConnector.getTempFileFromUrl(new URL(dataUrl), "xml"); - } - - @Test - public void testApiDataList() throws Exception { - Map filters = new TreeMap(); - filters.put("limit", "10"); - - Data datasets = client_read_test.dataList(filters); - assertTrue(datasets.getData() != null); - - assertTrue(datasets.getData().length == 10); - for (DataSet dataset : datasets.getData()) { - assertTrue("No qualities for dataset " + dataset.getDid(), dataset.getQualities() != null); - assertTrue("Not enough qualities for dataset " + dataset.getDid(), dataset.getQualities().length > 5); - } - } - - @Test - public void testApiAdditional() throws Exception { - DataQualityList dql = client_read_test.dataQualitiesList(); - assertTrue(dql.getQualities().length > 50); - } - - public void verifyCsvDataset(DataSet dataset) throws Exception { - int numInstances = (int) Double.parseDouble(dataset.getQualityMap().get("NumberOfInstances")); - int numFeatures = (int) Double.parseDouble(dataset.getQualityMap().get("NumberOfFeatures")); - - if (!dataset.getFormat().toLowerCase().equals("arff")) { - return; - } - - String fullUrl = url_test + "data/get_csv/" + dataset.getFileId() + "/" + dataset.getName() + ".csv"; - System.out.println(fullUrl); - final URL url = new URL(fullUrl); - final Reader reader = new InputStreamReader(url.openStream(), "UTF-8"); - final CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT); - try { - List records = parser.getRecords(); - int foundRecords = (int) parser.getRecordNumber() - 1; // -1 because of csv header - assertEquals(numInstances, foundRecords); - int foundColumns = records.get(0).size(); - assertEquals(numFeatures, foundColumns); - } finally { - parser.close(); - reader.close(); - } - } - - @Test - public void testGetDataAsCsvRandom() throws Exception { - Map filters = new TreeMap(); - filters.put("limit", "100"); - filters.put("number_instances", "10..10000"); - - DataSet[] all = client_read_test.dataList(filters).getData(); - - for (int i = 0; i < 10; ++i) { - DataSet current = all[i]; - verifyCsvDataset(current); - } - } - - @Test - public void testGetDataAsCsvSingle() throws Exception { - Map filters = new TreeMap(); - filters.put("data_id", "87"); - - DataSet[] all = client_read_test.dataList(filters).getData(); - for (DataSet current : all) { - verifyCsvDataset(current); - } - } - - // function that formats xml consistently, making it easy to compare them. - public static String toPrettyString(String xml, int indent) throws Exception { - // Turn xml string into a document - Document document = DocumentBuilderFactory.newInstance() - .newDocumentBuilder() - .parse(new InputSource(new ByteArrayInputStream(xml.getBytes("utf-8")))); - - // Remove whitespaces outside tags - XPath xPath = XPathFactory.newInstance().newXPath(); - NodeList nodeList = (NodeList) xPath.evaluate("//text()[normalize-space()='']", - document, - XPathConstants.NODESET); - - for (int i = 0; i < nodeList.getLength(); ++i) { - Node node = nodeList.item(i); - node.getParentNode().removeChild(node); - } - - // Setup pretty print options - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - transformerFactory.setAttribute("indent-number", indent); - Transformer transformer = transformerFactory.newTransformer(); - transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - - // Return pretty print xml string - StringWriter stringWriter = new StringWriter(); - transformer.transform(new DOMSource(document), new StreamResult(stringWriter)); - return stringWriter.toString(); - } -} diff --git a/apiconnector/src/test/java/apiconnector/TestEstimationProcedureFunctionality.java b/apiconnector/src/test/java/apiconnector/TestEstimationProcedureFunctionality.java deleted file mode 100644 index ad361de6..00000000 --- a/apiconnector/src/test/java/apiconnector/TestEstimationProcedureFunctionality.java +++ /dev/null @@ -1,26 +0,0 @@ -package apiconnector; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; -import org.openml.apiconnector.xml.EstimationProcedure; -import org.openml.apiconnector.xml.EstimationProcedures; - -import testbase.BaseTestFramework; - -public class TestEstimationProcedureFunctionality extends BaseTestFramework { - - @Test - public void testApiEstimationProcedureGet() throws Exception { - EstimationProcedure ep = client_read_test.estimationProcedureGet(1); - assertEquals(ep.getId(), 1); - assertEquals(ep.getTtid(), 1); - } - - @Test - public void testApiEstimationProcedureList() throws Exception { - EstimationProcedures ep = client_read_test.estimationProcedureList(); - assertTrue(ep.getEstimationProcedure().length > 10); - } -} diff --git a/apiconnector/src/test/java/apiconnector/TestFlowFunctionality.java b/apiconnector/src/test/java/apiconnector/TestFlowFunctionality.java deleted file mode 100644 index 85d84f78..00000000 --- a/apiconnector/src/test/java/apiconnector/TestFlowFunctionality.java +++ /dev/null @@ -1,154 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2017, Jan N. van Rijn - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package apiconnector; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.util.Arrays; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.junit.Test; -import org.openml.apiconnector.algorithms.Conversion; -import org.openml.apiconnector.io.ApiException; -import org.openml.apiconnector.xml.Flow; -import org.openml.apiconnector.xml.FlowTag; -import org.openml.apiconnector.xml.FlowUntag; -import org.openml.apiconnector.xml.Parameter; - -import testbase.BaseTestFramework; - -public class TestFlowFunctionality extends BaseTestFramework { - private static final int probe = 10; - - private static final String tag = "junittest"; - - @Test - public void testApiFlowDownload() throws Exception { - Flow flow = client_read_test.flowGet(probe); - - File tempXml = Conversion.stringToTempFile(xstream.toXML(flow), "flow", "xml"); - File tempXsd = client_read_test.getXSD("openml.implementation.upload"); - - System.out.println(Conversion.fileToString(tempXml)); - - assertTrue(Conversion.validateXML(tempXml, tempXsd)); - - // very easy checks, should all pass - assertTrue(flow.getId() == probe); - - } - - @Test - public void testApiFlowUpload() throws Exception { - - Integer uploaded_id = -1; - try { - Flow created = new Flow("test", "weka.classifiers.test.janistesting", "test", "test should be deleted", "english", "UnitTest"); - created.addComponent("B", new Flow("test2", "weka.classifiers.test.janistesting.subflow2", "test2", "test should be deleted", "english", "UnitTest") ); - created.addComponent("C", new Flow("test3", "weka.classifiers.test.janistesting.subflow3", "test3", "test should be deleted", "english", "UnitTest") ); - created.addComponent("D", new Flow("test4", "weka.classifiers.test.janistesting.subflow4", "test4", "test should be deleted", "english", "UnitTest") ); - - created.addParameter(new Parameter("test_a", "option", "bla1", "more bla1")); - created.addParameter(new Parameter("test_p", "option", "bla2", "more bla2")); - created.addParameter(new Parameter("test_q", "option", "blaq", "more blaqq")); - - uploaded_id = client_write_test.flowUpload(created); - FlowTag ft = client_write_test.flowTag(uploaded_id, tag); - assertTrue(Arrays.asList(ft.getTags()).contains(tag)); - FlowUntag fu = client_write_test.flowUntag(uploaded_id, tag); - assertTrue(fu.getTags() == null); - - Flow downloaded = client_read_test.flowGet(uploaded_id); - assertEquals(3, created.getParameter().length); - assertEquals(3, created.getComponent().length); - assertEquals(created.getParameter().length, downloaded.getParameter().length); - assertEquals(created.getComponent().length, downloaded.getComponent().length); - } catch (Exception e) { - e.printStackTrace(); - // possibly the previous test failed, and left the exact same flow on the server. - if (e instanceof ApiException) { - ApiException apiException = (ApiException) e; - if (apiException.getCode() == 171) { - int index = apiException.getMessage().indexOf("implementation_id") + "implementation_id".length()+1; - uploaded_id = Integer.parseInt(apiException.getMessage().substring(index)); - } - } - fail("Test failed: " + e.getMessage()); - } finally { - client_write_test.flowDelete(uploaded_id); - } - } - - @Test - public void testApiFlowUploadDuplicate() throws Exception { - Flow created = new Flow("test2", "weka.classifiers.test.javaunittest", "test", "test should be deleted", - "english", "UnitTest"); - created.addComponent("B", new Flow("test2", "weka.classifiers.test.janistesting", "test2", - "test should be deleted", "english", "UnitTest")); - created.addParameter(new Parameter("test_p", "option", "bla", "more bla")); - created.setCustom_name("Jans flow"); - int flowId = -1; - try { - flowId = client_write_test.flowUpload(created); - } catch (ApiException e) { - if (e.getCode() == 171) { - String idStr = e.getMessage().substring(e.getMessage().lastIndexOf(":") + 1); - flowId = Integer.parseInt(idStr); - client_admin_test.flowDelete(flowId); - fail("Flow was still on server, will be deleted now."); - } else { - fail("Flow was still on server, failed to delete it."); - } - } - try { - client_write_test.flowUpload(created); - fail("Test failed, flow upload should have been blocked."); - } catch (Exception e) { - // we expect an exception - } - client_write_test.flowDelete(flowId); - - } - - @Test - public void testUploadComplicatedFlow() throws Exception { - Random random = new Random(System.currentTimeMillis()); - String complicatedFlow = FileUtils.readFileToString(new File("data/FilteredClassifier_RandomForest.xml")) - .replace("{SENTINEL}", "SEN" + Math.abs(random.nextInt())); - Flow flow = (Flow) xstream.fromXML(complicatedFlow); - int flowId = client_write_test.flowUpload(flow); - client_write_test.flowDelete(flowId); - } -} diff --git a/apiconnector/src/test/java/apiconnector/TestRunFunctionality.java b/apiconnector/src/test/java/apiconnector/TestRunFunctionality.java deleted file mode 100644 index 8621aa3f..00000000 --- a/apiconnector/src/test/java/apiconnector/TestRunFunctionality.java +++ /dev/null @@ -1,222 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2017, Jan N. van Rijn - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package apiconnector; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -import org.junit.Ignore; -import org.junit.Test; -import org.openml.apiconnector.algorithms.Conversion; -import org.openml.apiconnector.io.ApiException; -import org.openml.apiconnector.xml.EvaluationRequest; -import org.openml.apiconnector.xml.EvaluationScore; -import org.openml.apiconnector.xml.Run; -import org.openml.apiconnector.xml.RunList; -import org.openml.apiconnector.xml.RunTag; -import org.openml.apiconnector.xml.RunUntag; -import org.openml.apiconnector.xml.Task; - -import testbase.BaseTestFramework; - -import static org.junit.Assert.*; - -public class TestRunFunctionality extends BaseTestFramework { - private static final int classif_task_id = 67; - private static final int curve_task_id = 763; // anneal - private static final int num_repeats = 1; - private static final int num_folds = 10; - private static final int num_samples = 9; // training set size approximately 900 - private static final String predictions_path = "data/predictions_task53.arff"; - private static final int FLOW_ID = 10; - - private static final String tag = "junittest"; - - @Test - public void testApiRunDownload() throws Exception { - - Run run = client_read_live.runGet(classif_task_id); - - File tempXml = Conversion.stringToTempFile(xstream.toXML(run), "run", "xml"); - File tempXsd = client_read_live.getXSD("openml.run.upload"); - - assertTrue(Conversion.validateXML(tempXml, tempXsd)); - - // very easy checks, should all pass - assertTrue(run.getRun_id() == classif_task_id); - - } - - @Test - public void testApiRunList() throws Exception { - List uploaderFilter = new ArrayList(); - uploaderFilter.add(1); - uploaderFilter.add(16); - - Map> filters = new HashMap>(); - filters.put("uploader", uploaderFilter); - - RunList rl = client_read_live.runList(filters, 100, 0); - assertTrue(rl.getRuns().length == 100); - - for (Run r : rl.getRuns()) { - assertTrue(uploaderFilter.contains(r.getUploader())); - } - - } - - @Test - public void testApiRunUpload() throws Exception { - String[] tags = {"first_tag", "another_tag"}; - - Run r = new Run(classif_task_id, null, FLOW_ID, null, null, tags); - - for (int i = 0; i < num_repeats; ++i) { - for (int j = 0; j < num_folds; ++j) { - r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", i, j, null, null)); - } - } - - File predictions = new File(predictions_path); - - Map output_files = new HashMap(); - - output_files.put("predictions", predictions); - - int runId = client_write_test.runUpload(r, output_files); - - Run newrun = client_write_test.runGet(runId); - - Set uploadedTags = new HashSet(Arrays.asList(newrun.getTag())); - Set providedTags = new HashSet(Arrays.asList(tags)); - - assertTrue(uploadedTags.equals(providedTags)); - - RunTag rt = client_write_test.runTag(runId, tag); - assertTrue(Arrays.asList(rt.getTags()).contains(tag)); - RunUntag ru = client_write_test.runUntag(runId, tag); - assertTrue(Arrays.asList(ru.getTags()).contains(tag) == false); - - client_write_test.runDelete(runId); - } - - @Test - public void testApiRunUploadIllegalMeasure() throws Exception { - Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); - r.addOutputEvaluation(new EvaluationScore("unexisting", 1.0, "[1.0, 1.0]", 0, 0, null, null)); - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.runUpload(r, null) - ); - assertEquals("Error processing output data: unknown evaluation measure: Measure(s): unexisting", thrown.getMessage()); - } - - @Test - public void testApiRunUploadWronglyParameterziedMeasureRepeats() throws Exception { - Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); - r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", num_repeats, 0, null, null)); - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.runUpload(r, null) - ); - assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(1, 0)", thrown.getMessage()); - } - - @Test - public void testApiRunUploadWronglyParameterziedMeasureFolds() throws Exception { - Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); - r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, num_folds, null, null)); - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.runUpload(r, null) - ); - assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 10)", thrown.getMessage()); - } - - @Test - public void testApiRunUploadWronglyParameterziedMeasureSample() throws Exception { - Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); - r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, 0, 0, 0)); - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.runUpload(r, null) - ); - assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 0, 0)", thrown.getMessage()); - } - - @Test - public void testApiRunUploadWronglyParameterziedMeasureSampleCurveTask() throws Exception { - Run r = new Run(curve_task_id, null, FLOW_ID, null, null, null); - r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, 0, num_samples, null)); - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.runUpload(r, null) - ); - assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 0, 9)", thrown.getMessage()); - } - - - @Test - public void testApiRunUploadSamples() throws Exception { - Run r = new Run(curve_task_id, null, FLOW_ID, null, null, null); - for (int i = 0; i < num_samples; ++i) { - r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, 0, i, null)); - client_write_test.runUpload(r, null); - } - } - - @Test - @Ignore // skipping because of test server state - public void testApiEvaluationRequest() throws Exception { - // this test assumes that there are runs on the test server. - // might not be the case just after reset - - // gives evaluation id "42", which does not exist. - // therefore we get an actual run that is not evaluated by this engine back. - Map filters = new TreeMap(); - Integer[] ttids = {3,4}; - String ttidString = Arrays.toString(ttids).replaceAll(" ", "").replaceAll("\\[", "").replaceAll("\\]", ""); - int numRequests = 25; - filters.put("ttid", ttidString); - EvaluationRequest er = client_admin_test.evaluationRequest(42, "random", numRequests, filters); - assertTrue(er.getRuns().length == numRequests); - Run r = client_write_test.runGet(er.getRuns()[0].getRun_id()); - Task t = client_write_test.taskGet(r.getTask_id()); - assertTrue(Arrays.asList(ttids).contains(t.getTask_type_id())); - } -} diff --git a/apiconnector/src/test/java/apiconnector/TestSetupFunctions.java b/apiconnector/src/test/java/apiconnector/TestSetupFunctions.java deleted file mode 100644 index 53c6d76d..00000000 --- a/apiconnector/src/test/java/apiconnector/TestSetupFunctions.java +++ /dev/null @@ -1,138 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2017, Jan N. van Rijn - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package apiconnector; - -import java.io.File; -import java.net.URL; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.commons.io.FileUtils; -import org.junit.Test; -import org.openml.apiconnector.algorithms.Conversion; -import org.openml.apiconnector.io.ApiException; -import org.openml.apiconnector.io.OpenmlConnector; -import org.openml.apiconnector.xml.Run; -import org.openml.apiconnector.xml.SetupExists; -import org.openml.apiconnector.xml.SetupTag; -import org.openml.apiconnector.xml.SetupUntag; -import org.openml.apiconnector.xstream.XstreamXmlMapping; - -import com.thoughtworks.xstream.XStream; - -import testbase.BaseTestFramework; - -import static org.junit.Assert.*; - -public class TestSetupFunctions extends BaseTestFramework { - // TODO: Note that this test case runs on live. we should add - // runs to test server to alliviate this. - private static final String url = "https://www.openml.org/"; // Lookup test, can be done live - private static final String session_hash = "8baa83ecddfe44b561fd3d92442e3319"; - private static final OpenmlConnector client_write = new OpenmlConnector(url,session_hash); // TODO: write account - private static final XStream xstream = XstreamXmlMapping.getInstance(); - private static final String tag = "junittest"; - - @Test - public void testFindRightSetup() throws Exception { - Integer[] run_ids = {541980, 541944}; - - for (Integer run_id : run_ids) { - Run r = client_write.runGet(run_id); - int setup_id = r.getSetup_id(); - - File description = getDescriptionFile(r, run_id); - SetupExists se = client_write.setupExists(description); - assertTrue(se.exists()); - assertEquals(se.getId(), setup_id); - - try { - SetupTag st = client_write.setupTag(setup_id, tag); - assertTrue(Arrays.asList(st.getTags()).contains(tag)); - } catch(ApiException ae) { - // tolerate. - assertTrue(ae.getMessage().startsWith("Entity already tagged by this tag. ")); - } - SetupUntag su = client_write.setupUntag(setup_id, tag); - assertFalse(Arrays.asList(su.getTags()).contains(tag)); - } - } - - @Test - public void testFindNonexistingSetup() throws Exception { - Integer[] run_ids = {541980, 541944}; - Map searchReplace = new TreeMap(); - searchReplace.put("500", "bla2"); // matches run 541980 - searchReplace.put("weka.classifiers.trees.REPTree", "weka.classifiers.trees.J50 -C 0.25 -M 2"); // matches run 541944 - - for (Integer run_id : run_ids) { - Run r = client_write.runGet(run_id); - - File description = getDescriptionFile(r, run_id); - - // adjust the run file - Path path = Paths.get(description.getAbsolutePath()); - Charset charset = StandardCharsets.UTF_8; - - String content = new String(Files.readAllBytes(path), charset); - for (String search : searchReplace.keySet()) { - content = content.replaceAll(search, searchReplace.get(search)); - } - Files.write(path, content.getBytes(charset)); - SetupExists se = client_write.setupExists(description); - // TODO(Jan van Rijn): this was assertFalse, but returns true. Since this description is unchanged, do we - // expect a true? - assertTrue(se.exists()); - - // now try it with empty run file - Run rEmpty = new Run(null, null, r.getFlow_id(), null, null, null); - File runEmpty = Conversion.stringToTempFile(xstream.toXML(rEmpty), "openml-retest-run" + run_id, "xml"); - - SetupExists se2 = client_write.setupExists(runEmpty); - assertFalse(se2.exists()); - } - } - - private static File getDescriptionFile(Run r, int run_id) throws Exception { - Integer descriptionFileId = r.getOutputFileAsMap().get("description").getFileId(); - URL descriptionUrl = client_write.getOpenmlFileUrl(descriptionFileId, "description_run" + run_id + ".xml"); - File description = File.createTempFile("description_run" + run_id, ".xml"); - FileUtils.copyURLToFile(descriptionUrl, description); - return description; - } - -} diff --git a/apiconnector/src/test/java/apiconnector/TestStudyFunctions.java b/apiconnector/src/test/java/apiconnector/TestStudyFunctions.java deleted file mode 100644 index d66428ec..00000000 --- a/apiconnector/src/test/java/apiconnector/TestStudyFunctions.java +++ /dev/null @@ -1,120 +0,0 @@ -package apiconnector; - -import static org.junit.Assert.*; - -import java.util.Arrays; -import java.util.Map; -import java.util.TreeMap; - -import org.junit.Ignore; -import org.junit.Test; -import org.openml.apiconnector.xml.Study; -import org.openml.apiconnector.xml.StudyAttach; -import org.openml.apiconnector.xml.StudyDetach; -import org.openml.apiconnector.xml.StudyList; - -import testbase.BaseTestFramework; - -public class TestStudyFunctions extends BaseTestFramework { - - @Test - public void testApiGetStudy() throws Exception { - Study s = client_read_live.studyGet(34); - assertTrue(s.getDataset().length == 105); - assertTrue(s.getFlows().length == 27); - assertTrue(s.getTasks().length == 105); - assertTrue(s.getSetups().length == 30); - } - - @Test - public void testApiGetStudyData() throws Exception { - Study s = client_read_live.studyGet(34, "data"); - assertTrue(s.getDataset().length > 5); - assertTrue(s.getFlows() == null); - assertTrue(s.getTasks() == null); - assertTrue(s.getSetups() == null); - } - - @Test - public void testApiGetStudyTasks() throws Exception { - Study s = client_read_live.studyGet(34, "tasks"); - assertTrue(s.getDataset() == null); - assertTrue(s.getFlows() == null); - assertTrue(s.getTasks().length > 5); - assertTrue(s.getSetups() == null); - } - - @Test - public void testApiGetStudyFlows() throws Exception { - Study s = client_read_live.studyGet(34, "flows"); - assertTrue(s.getTag().length > 0); - assertTrue(s.getDataset() == null); - assertTrue(s.getFlows().length > 5); - assertTrue(s.getTasks() == null); - assertTrue(s.getSetups() == null); - } - - @Test - public void testApiGetStudySetups() throws Exception { - Study s = client_read_live.studyGet(34, "setups"); - assertTrue(s.getDataset() == null); - assertTrue(s.getFlows() == null); - assertTrue(s.getTasks() == null); - assertTrue(s.getSetups().length > 5); - } - - @Test - public void testApiGetStudyByAlias() throws Exception { - Study s = client_read_test.studyGet("OpenML100", "tasks"); - assertTrue(s.getDataset().length > 10); - assertTrue(s.getFlows() == null); - assertTrue(s.getTasks().length > 10); - assertTrue(s.getSetups() == null); - } - - @Test - public void getStudy() throws Exception { - Study study = client_read_test.studyGet(1); - assertTrue(study.getTasks().length > 0); - assertTrue(study.getDataset().length > 0); - assertNull(study.getSetups()); - assertNull(study.getFlows()); - assertNull(study.getRuns()); - assertNull(study.getBenchmark_suite()); - assertEquals("task", study.getMain_entity_type()); - } - - @Test - public void uploadStudy() throws Exception { - Integer[] taskIdsInitial = {1, 2, 3}; - Integer[] taskIdsAdditional = {4, 5, 6}; - Study study = new Study(null, "test", "test", null, taskIdsInitial, null); - int studyId = client_write_test.studyUpload(study); - - Study studyDownload = client_read_test.studyGet(studyId); - assertArrayEquals(taskIdsInitial, studyDownload.getTasks()); - - StudyAttach sa = client_write_test.studyAttach(studyId, Arrays.asList(taskIdsAdditional)); - assertTrue(taskIdsInitial.length + taskIdsAdditional.length == sa.getLinkedEntities()); - - StudyDetach sd = client_write_test.studyDetach(studyId, Arrays.asList(taskIdsInitial)); - assertTrue(taskIdsAdditional.length == sd.getLinkedEntities()); - - Study studyDownload2 = client_read_test.studyGet(studyId); - assertArrayEquals(taskIdsAdditional, studyDownload2.getTasks()); - } - - @Test - @Ignore - public void studyList() throws Exception { - Map filters = new TreeMap(); - filters.put("status", "all"); - filters.put("limit", "20"); - StudyList sl = client_read_test.studyList(filters); - assertTrue(sl.getStudies().length > 5); - - for (Study s : sl.getStudies()) { - assertTrue(s.getId() > 0); - } - } -} diff --git a/apiconnector/src/test/java/apiconnector/TestTaskFunctions.java b/apiconnector/src/test/java/apiconnector/TestTaskFunctions.java deleted file mode 100644 index 4b51960e..00000000 --- a/apiconnector/src/test/java/apiconnector/TestTaskFunctions.java +++ /dev/null @@ -1,339 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2017, Jan N. van Rijn - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package apiconnector; - -import java.io.File; -import java.net.URL; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import org.json.JSONArray; -import org.junit.Ignore; -import org.junit.Test; -import org.openml.apiconnector.algorithms.Conversion; -import org.openml.apiconnector.algorithms.TaskInformation; -import org.openml.apiconnector.io.ApiException; -import org.openml.apiconnector.io.HttpConnector; -import org.openml.apiconnector.xml.DataQuality; -import org.openml.apiconnector.xml.DataSetDescription; -import org.openml.apiconnector.xml.Task; -import org.openml.apiconnector.xml.TaskInputs; -import org.openml.apiconnector.xml.Tasks; -import org.openml.apiconnector.xml.TaskInputs.Input; - -import testbase.BaseTestFramework; - -import static org.junit.Assert.*; - -public class TestTaskFunctions extends BaseTestFramework { - - public static final Integer[] TASK_ILLEGAL_INPUT_CODES = {621, 622}; - - private static final Integer taskId = 1; - - @Test - public void testApiAdditional() throws Exception { - Task t = client_read_test.taskGet(taskId); - - int epId = TaskInformation.getEstimationProcedure(t).getId(); - client_read_test.estimationProcedureGet(epId); - - TaskInputs ti = client_read_test.taskInputs(taskId); - assertTrue(ti.getInputsAsMap().size() > 2); - - URL splitsUrl = TaskInformation.getEstimationProcedure(t).getData_splits_url(); - - Integer dataId = TaskInformation.getSourceData(t).getData_set_id(); - File splitsFile = HttpConnector.getTempFileFromUrl(splitsUrl, "arff"); - String[] splits = Conversion.fileToString(splitsFile).split("\n"); - DataQuality dq = client_read_test.dataQualities(dataId, null); - int numInstances = dq.getQualitiesMap().get("NumberOfInstances").intValue(); - - assertTrue(splits.length > numInstances); // basic check - - } - - @Test - public void testTaskCreationNoInputs() throws Exception { - Input estimation_procedure = new Input("estimation_procedure", "1"); - Input data_set = new Input("source_data", "1"); - Input target_feature = new Input("target_feature", "class"); - Input[] inputs = { estimation_procedure, data_set, target_feature }; - - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)) - ); - assertTrue(thrown.getMessage().startsWith("Task already exists.")); - } - - @Test - public void testTaskCreationIllegalValues() throws Exception { - Input estimation_procedure = new Input("estimation_procedure", "15"); - Input data_set = new Input("illegal_source_data", "-1"); - Input target_feature = new Input("target_feature", "class"); - Input[] inputs = { estimation_procedure, data_set, target_feature }; - - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)) - ); - assertEquals("Task contains illegal inputs.: problematic input: illegal_source_data", thrown.getMessage()); - } - - @Test - public void testTaskCreationDuplicateValue() throws Exception { - Input estimation_procedure = new Input("estimation_procedure", "15"); - Input data_set = new Input("source_data", "1"); - Input data_set2 = new Input("source_data", "1"); - Input target_feature = new Input("target_feature", "class"); - Input[] inputs = { estimation_procedure, data_set, target_feature, data_set2 }; - - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)) - ); - assertEquals("Task contains duplicate inputs.: problematic input: source_data", thrown.getMessage()); - } - - @Test - public void testTaskCreationNotRequiredValues() throws Exception { - Input estimation_procedure = new Input("estimation_procedure", "15"); - Input target_feature = new Input("target_feature", "class"); - Input[] inputs = { estimation_procedure, target_feature }; - - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)) - ); - assertEquals("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs: []", thrown.getMessage()); - } - - @Test - public void testTaskAlreadyExists() throws Exception { - // try to create it twice, as it might not exists yet the first time. - ApiException thrown = assertThrows( - ApiException.class, - () -> { - for (int i = 0; i < 2; ++i) { - Input estimation_procedure = new Input("estimation_procedure", "1"); - Input data_set = new Input("source_data", "1"); - Input target_feature = new Input("target_feature", "class"); - Input[] inputs = { estimation_procedure, data_set, target_feature }; - client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)); - } - } - ); - assertTrue(thrown.getMessage().startsWith("Task already exists.")); - } - - @Test - public void testCreateTaskIllegalValues() throws Exception { - Random random = new Random(); - - Input[] inputs = new Input[4]; - inputs[0] = new Input("estimation_procedure", "1"); - inputs[1] = new Input("source_data", "2"); - inputs[2] = new Input("target_feature", "class"); - inputs[3] = new Input("evaluation_measures", "predictive_accuracy"); - - for (int i = 0; i < 4; ++i) { - Input[] derived = Arrays.copyOf(inputs, inputs.length); - derived[i] = new Input(derived[i].getName(), derived[i].getValue() + "_" + random.nextInt()); - try { - client_write_test.taskUpload(new TaskInputs(null, 1, derived, null)); - // previous statement should not terminate without ApiException. - throw new Exception("Tasks did not get blocked by Api"); - } catch(ApiException e) { - assertTrue(Arrays.asList(TASK_ILLEGAL_INPUT_CODES).contains(e.getCode())); - } - } - } - - @Test - public void testAvoidInactiveDataset() throws Exception { - File toUpload = new File(TestDataFunctionality.data_file); - DataSetDescription dsd = new DataSetDescription("test", "Unit test should be deleted", "arff", "class"); - int dataId = client_write_test.dataUpload(dsd, toUpload); - - Input[] inputs = new Input[3]; - inputs[0] = new Input("estimation_procedure", "1"); - inputs[1] = new Input("source_data", "" + dataId); - inputs[2] = new Input("target_feature", "class"); - - try { - client_write_test.taskUpload(new TaskInputs(null, 6, inputs, null)); - fail("Should not be able to upload task."); - } catch(Exception e) { } - } - - @Test - public void testAvoidInactiveDatasetClustering() throws Exception { - // clustering task is more complex test case, since the dataset does not need to be processed. - File toUpload = new File(TestDataFunctionality.data_file); - DataSetDescription dsd = new DataSetDescription("test", "Unit test should be deleted", "arff", "class"); - int dataId = client_write_test.dataUpload(dsd, toUpload); - - Input[] inputs = new Input[2]; - inputs[0] = new Input("estimation_procedure", "17"); - inputs[1] = new Input("source_data", "" + dataId); - - try { - client_write_test.taskUpload(new TaskInputs(null, 5, inputs, null)); - fail("Should not be able to upload task."); - } catch(Exception e) { } - } - - @Test - public void testCreateChallengeTask() throws Exception { - Input[] inputs = new Input[4]; - inputs[0] = new Input("estimation_procedure", "18"); - inputs[1] = new Input("source_data", "128"); - inputs[2] = new Input("source_data_labeled", "129"); - inputs[3] = new Input("target_feature", "class"); - - int taskId = 0; - try { - taskId = client_write_test.taskUpload(new TaskInputs(null, 6, inputs, null)); - } catch (ApiException e) { - taskId = TaskInformation.getTaskIdsFromErrorMessage(e)[0]; - throw e; - } finally { - client_write_test.taskDelete(taskId); - } - } - - @Test - public void testCreateTaskWithCostMatrix() throws Exception { - JSONArray costMatrixOrig = new JSONArray("[[0, 1], [10, 0]]"); - - Input[] inputs = new Input[4]; - inputs[0] = new Input("estimation_procedure", "1"); - inputs[1] = new Input("source_data", "2"); - inputs[2] = new Input("target_feature", "class"); - inputs[3] = new Input("cost_matrix", costMatrixOrig.toString()); - - int uploadId = 0; - try { - uploadId = client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)); - Task downloaded = client_read_test.taskGet(uploadId); - JSONArray costMatrixDownloaded = TaskInformation.getCostMatrix(downloaded); - assertEquals(costMatrixOrig.toString(), costMatrixDownloaded.toString()); - } catch (ApiException e) { - uploadId = TaskInformation.getTaskIdsFromErrorMessage(e)[0]; - throw e; - } finally { - client_write_test.taskDelete(uploadId); - } - } - - - @Test - public void testCreateClassificationTaskNumericTarget() throws Exception { - Input[] inputs = new Input[3]; - inputs[0] = new Input("estimation_procedure", "1"); - inputs[1] = new Input("source_data", "1"); - inputs[2] = new Input("target_feature", "carbon"); - - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)) - ); - assertTrue(thrown.getMessage().startsWith("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs")); - } - - @Test - public void testCreateRegressionTaskNominalTarget() throws Exception { - Input[] inputs = new Input[3]; - inputs[0] = new Input("estimation_procedure", "7"); - inputs[1] = new Input("source_data", "1"); - inputs[2] = new Input("target_feature", "class"); - - - ApiException thrown = assertThrows( - ApiException.class, - () -> client_write_test.taskUpload(new TaskInputs(null, 2, inputs, null)) - ); - assertEquals("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs: [carbon, hardness, strength, thick, width, len]", thrown.getMessage()); - } - - @Test - public void testCreateTask() throws Exception { - Integer uploadId1 = null; - Integer uploadId2 = null; - - Input estimation_procedure = new Input("estimation_procedure", "1"); - Input data_set = new Input("source_data", "2"); - Input target_feature = new Input("target_feature", "bkblk"); // some random attribute that is unlikely to have tasks - Input evaluation_measure = new Input("evaluation_measures", "predictive_accuracy"); - - try { - // create task object - Input[] inputs = { estimation_procedure, data_set, target_feature, evaluation_measure}; - try { - // try catch for deleting tasks that were already on the server - uploadId1 = client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)); - } catch(ApiException e) { - uploadId1 = TaskInformation.getTaskIdsFromErrorMessage(e)[0]; - throw e; - } - - // create task similar task object (with one value less) - Input[] inputs2 = { estimation_procedure, data_set, target_feature }; - try { - // try catch for deleting tasks that were already on the server - uploadId2 = client_write_test.taskUpload(new TaskInputs(null, 1, inputs2, null)); - } catch(ApiException e) { - uploadId2 = TaskInformation.getTaskIdsFromErrorMessage(e)[0]; - throw e; - } - } finally { - // make sure that the task does not exists anymore - if (uploadId1 != null) { try { client_write_test.taskDelete(uploadId1); } catch(ApiException a) {} } - if (uploadId2 != null) { try { client_write_test.taskDelete(uploadId2); } catch(ApiException a) {} } - } - } - - @Test - public void testApiTaskList() throws Exception { - Map filters = new HashMap(); - filters.put("type", "1"); - filters.put("limit", "100"); - Tasks tasks = client_read_test.taskList(filters); - assertTrue(tasks.getTask().length == 100); - for (org.openml.apiconnector.xml.Tasks.Task t : tasks.getTask()) { - // assertTrue(t.getQualities().length > 5); - assertTrue(t.getInputs().length > 2); - } - } -} diff --git a/apiconnector/src/test/java/apiconnector/TestUploadArff.java b/apiconnector/src/test/java/apiconnector/TestUploadArff.java deleted file mode 100644 index 393ca778..00000000 --- a/apiconnector/src/test/java/apiconnector/TestUploadArff.java +++ /dev/null @@ -1,61 +0,0 @@ -package apiconnector; - -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; - -import org.junit.Test; -import org.openml.apiconnector.xml.DataSetDescription; - -import testbase.BaseTestFramework; - -public class TestUploadArff extends BaseTestFramework { - - private static final String DATASETPATH = "data" + File.separator + "arff_test" + File.separator; - - @Test - public void testUploadDataset() throws IOException { - // Test XML description - Path path = Paths.get(DATASETPATH); - // Pass through each dataset on the directory - Files.walkFileTree(path, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attr) throws IOException { - if (VERBOSE) { - System.out.println("ARFF-check processing " + file.getFileName()); - } - File toUpload = new File(file.toString()); - int id = -1; - // boolean to signal the validity of a dataset - boolean invalid = file.getFileName().toString().startsWith("invalid"); - - try { - DataSetDescription dsd = new DataSetDescription("test", "Unit test should be deleted", "arff", "class"); - id = client_write_test.dataUpload(dsd, toUpload); - // Only reached by a dataset (ARFF file) that gets uploaded. - client_write_test.dataDelete(id); - } catch(Exception e) { - if (VERBOSE) { - e.printStackTrace(); - } - } finally { - if(invalid) { - assertTrue(id == -1); - } else { - assertTrue(id != -1); - } - // Reset the dataset id - id = -1; - } - // Keep going through datasets - return FileVisitResult.CONTINUE; - }}); - } -} diff --git a/apiconnector/src/test/java/apiconnector/TestUserTypePrivileges.java b/apiconnector/src/test/java/apiconnector/TestUserTypePrivileges.java deleted file mode 100644 index 8b003add..00000000 --- a/apiconnector/src/test/java/apiconnector/TestUserTypePrivileges.java +++ /dev/null @@ -1,269 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2017, Jan N. van Rijn - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package apiconnector; - -import static org.junit.Assert.*; - -import java.io.File; -import java.io.IOException; - -import org.junit.Test; -import org.openml.apiconnector.io.ApiException; -import org.openml.apiconnector.xml.DataFeature; -import org.openml.apiconnector.xml.DataQuality; -import org.openml.apiconnector.xml.DataQuality.Quality; - -import testbase.BaseTestFramework; - -import org.openml.apiconnector.xml.DataSetDescription; -import org.openml.apiconnector.xml.Flow; -import org.openml.apiconnector.xml.Run; -import org.openml.apiconnector.xml.RunEvaluation; -import org.openml.apiconnector.xml.RunTrace; -import org.openml.apiconnector.xml.TaskInputs; - -public class TestUserTypePrivileges extends BaseTestFramework { - - private static final String data_file = "data/iris.arff"; - - private static final Integer EVAL_ID = 2; - - private static final int PRIVATE_DATASET_ID = 130; - - @Test(expected=ApiException.class) - public void testApiDataQualityUpload() throws Exception { - DataQuality dq = new DataQuality(1, EVAL_ID, new Quality[0]); - try { - client_write_test.dataQualitiesUpload(dq); - } catch(ApiException e) { - assertTrue(e.getCode() == 106); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiAttemptDownloadPrivateDataset() throws Exception { - client_read_test.dataGet(PRIVATE_DATASET_ID); - } - - @Test(expected=ApiException.class) - public void testApiAttemptDownloadPrivateDataFeatures() throws Exception { - client_read_test.dataFeatures(PRIVATE_DATASET_ID); - } - - @Test(expected=ApiException.class) - public void testApiAttemptDownloadPrivateDataQualities() throws Exception { - client_read_test.dataQualities(PRIVATE_DATASET_ID, null); - } - - @Test(expected=IOException.class) - public void testApiAttemptDownloadPrivateDataFile() throws Exception { - DataSetDescription dsd = client_admin_test.dataGet(PRIVATE_DATASET_ID); - client_read_test.datasetGet(dsd); - } - - @Test - public void testApiAdminAttemptDownloadPrivateDataset() throws Exception { - DataSetDescription dsd = client_admin_test.dataGet(PRIVATE_DATASET_ID); - client_admin_test.dataFeatures(PRIVATE_DATASET_ID); - client_admin_test.dataQualities(PRIVATE_DATASET_ID, null); - client_admin_test.datasetGet(dsd); - } - - public void testApiAdminDownloadPrivateDataset() throws Exception { - client_admin_test.dataGet(PRIVATE_DATASET_ID); - } - - public void testApiAdminDownloadPrivateDataFeatures() throws Exception { - client_admin_test.dataFeatures(PRIVATE_DATASET_ID); - } - - public void testApiAdminDownloadPrivateDataQualities() throws Exception { - client_admin_test.dataQualities(PRIVATE_DATASET_ID, null); - } - - public void testApiAdminDownloadPrivateDataFile() throws Exception { - DataSetDescription dsd = client_admin_test.dataGet(PRIVATE_DATASET_ID); - client_admin_test.datasetGet(dsd); - } - - @Test(expected=ApiException.class) - public void testApiDataFeatureUpload() throws Exception { - DataFeature df = new DataFeature(1, EVAL_ID, new DataFeature.Feature[0]); - try { - client_write_test.dataFeaturesUpload(df); - } catch(ApiException e) { - assertTrue(e.getCode() == 106); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiRunEvaluationUpload() throws Exception { - RunEvaluation re = new RunEvaluation(1, 1); - try { - client_write_test.runEvaluate(re); - } catch(ApiException e) { - assertTrue(e.getCode() == 106); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiRunTraceUpload() throws Exception { - RunTrace rt = new RunTrace(1); - try { - client_write_test.runTraceUpload(rt); - } catch(ApiException e) { - assertTrue(e.getCode() == 106); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiDataUpload() throws Exception { - DataSetDescription dsd = new DataSetDescription("test", "Unit test should be deleted", "arff", "class"); - try { - client_read_test.dataUpload(dsd, new File(data_file)); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiStatusActivate() throws Exception { - DataSetDescription dsd = new DataSetDescription("test", "Unit test should be deleted", "arff", "class"); - int dataId = client_write_test.dataUpload(dsd, new File(data_file)); - try { - client_write_test.dataStatusUpdate(dataId, "active"); - } catch(ApiException e) { - assertTrue(e.getCode() == 696); - throw e; - } - client_admin_test.dataStatusUpdate(dataId, "active"); - client_write_test.dataStatusUpdate(dataId, "deactivated"); - client_admin_test.dataStatusUpdate(dataId, "active"); - } - - @Test(expected=ApiException.class) - public void testApiDataTag() throws Exception { - try { - client_read_test.dataTag(1, "default_tag"); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiDataUntag() throws Exception { - try { - client_read_test.dataUntag(1, "default_tag"); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiDataDelete() throws Exception { - try { - client_read_test.dataDelete(1); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiFlowUpload() throws Exception { - Flow flow = new Flow("test2", "weka.classifiers.test.javaunittest", "test", "test should be deleted", "english", "UnitTest"); - try { - client_read_test.flowUpload(flow); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiTaskUpload() throws Exception { - TaskInputs task = new TaskInputs(1, 1, null, null); - try { - client_read_test.taskUpload(task); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiRunUpload() throws Exception { - Run run = new Run(1, null, 1, null, null, null); - try { - client_read_test.runUpload(run, null); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiFlowDelete() throws Exception { - try { - client_read_test.flowDelete(1); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiTaskDelete() throws Exception { - try { - client_read_test.taskDelete(1); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } - - @Test(expected=ApiException.class) - public void testApiRunDelete() throws Exception { - try { - client_read_test.runDelete(1); - } catch(ApiException e) { - assertTrue(e.getCode() == 104); - throw e; - } - } -} diff --git a/apiconnector/src/test/java/examples/TestPaperOpenMLBenchmarkSuites.java b/apiconnector/src/test/java/examples/TestPaperOpenMLBenchmarkSuites.java deleted file mode 100644 index b7cab2eb..00000000 --- a/apiconnector/src/test/java/examples/TestPaperOpenMLBenchmarkSuites.java +++ /dev/null @@ -1,75 +0,0 @@ -package examples; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertTrue; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.junit.Test; -import org.openml.apiconnector.io.OpenmlConnector; -import org.openml.apiconnector.xml.RunList; -import org.openml.apiconnector.xml.Study; -import org.openml.apiconnector.xml.StudyList; -import org.openml.apiconnector.xml.Tasks; - -import testbase.BaseTestFramework; - -public class TestPaperOpenMLBenchmarkSuites extends BaseTestFramework { - - // alias - protected static final OpenmlConnector openml = client_write_test; - - @Test - public void listBenchmarksuites() throws Exception { - Map filters = new TreeMap(); - filters.put("status", "all"); - filters.put("main_entity_type", "task"); - filters.put("limit", "20"); - StudyList list = openml.studyList(filters); - - assertTrue(list.getStudies().length > 0); - } - - @Test - public void attachDetachStudy() throws Exception { - // find 250 tasks that we are interested in, e.g., the tasks that have between - // 100 and 10000 instances and between 4 and 20 attributes - Map filtersOrig = new TreeMap(); - filtersOrig.put("number_instances", "100..10000"); - filtersOrig.put("number_features", "4..20"); - filtersOrig.put("limit", "250"); - Tasks tasksOrig = openml.taskList(filtersOrig); - - // create the study - Study study = new Study(null, "test", "test", null, tasksOrig.getTaskIds(), null); - int studyId = openml.studyUpload(study); - - // until the benchmark suite is activated, we can also add some more tasks. Search for the letter dataset: - Map filtersAdd = new TreeMap(); - filtersAdd.put("data_name", "letter"); - filtersAdd.put("limit", "1"); - Tasks tasksAdd = openml.taskList(filtersAdd); - openml.studyAttach(studyId, Arrays.asList(tasksAdd.getTaskIds())); - - // or even remove these again - openml.studyDetach(studyId, Arrays.asList(tasksAdd.getTaskIds())); - - // download the study - Study studyDownloaded = openml.studyGet(studyId); - assertArrayEquals(tasksOrig.getTaskIds(), studyDownloaded.getTasks()); - } - - @Test - public void downloadResultsBenchmarkSuite() throws Exception { - Study benchmarkSuite = openml.studyGet("OpenML100", "tasks"); - - Map> filters = new TreeMap>(); - filters.put("task", Arrays.asList(benchmarkSuite.getTasks())); - RunList rl = openml.runList(filters, 200, null); - - assertTrue(rl.getRuns().length > 0); - } -} diff --git a/apiconnector/src/test/java/examples/TestThesisMassivelyCollaborativeMachineLearning.java b/apiconnector/src/test/java/examples/TestThesisMassivelyCollaborativeMachineLearning.java deleted file mode 100644 index a8d93c97..00000000 --- a/apiconnector/src/test/java/examples/TestThesisMassivelyCollaborativeMachineLearning.java +++ /dev/null @@ -1,312 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2017, Jan N. van Rijn - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package examples; - -import static org.junit.Assert.assertTrue; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.junit.Ignore; -import org.junit.Test; -import org.openml.apiconnector.io.OpenmlConnector; -import org.openml.apiconnector.xml.EvaluationList; -import org.openml.apiconnector.xml.EvaluationList.Evaluation; -import org.openml.apiconnector.xml.Flow; -import org.openml.apiconnector.xml.Parameter; -import org.openml.apiconnector.xml.SetupParameters; -import org.openml.apiconnector.xml.Study; - -import testbase.BaseTestFramework; - -public class TestThesisMassivelyCollaborativeMachineLearning extends BaseTestFramework { - - // alias - protected static final OpenmlConnector openml = client_read_live; - - private static final int evaluationLimit = 100; - - @Test - @Ignore - public void testCompareSetups() throws Exception { - Study study = openml.studyGet(34); - String measure = "predictive_accuracy"; - File resultFile = new File("results.csv"); - assertTrue(study != null); - assertTrue(study.getTasks() != null); - assertTrue(study.getSetups() != null); - compareClassifiersAcrossTasks(openml, - Arrays.asList(study.getTasks()), - Arrays.asList(study.getSetups()), - measure, resultFile); - } - - @Test - @Ignore - public void testFlowsOnTaks() throws Exception { - int taskId = 6; - String measure = "predictive_accuracy"; - File resultFile = new File("flowsOnTask" + taskId + ".csv"); - int resultLimit = 20; - - flowsOnTask(openml, taskId, measure, resultFile, resultLimit); - } - - @Test - @Ignore // takes too long - public void testHyperparameterEffect() throws Exception { - Integer[] taskIds = {6, 21, 28, 41, 58}; - Integer[] setupIds = {8920, 8921, 8922, 8923, 8924, 8925, 8926, 8927, 8929, 8930, 8932, 8937, 8938}; - String measure = "predictive_accuracy"; - String hyperparameter = "weka.RBFKernel(4)_G"; - File resultFile = new File("effectG.csv"); - - hyperparameterEffect(openml, Arrays.asList(taskIds), Arrays.asList(setupIds), hyperparameter, measure, resultFile); - } - - public static void compareClassifiersAcrossTasks(OpenmlConnector openml, List taskIds, List setupIds, String evaluationMeasure, File resultsFile) throws Exception { - // obtains all evaluations that comply to the three filters - EvaluationList results = openml.evaluationList(taskIds, setupIds, evaluationMeasure, evaluationLimit); - // initialize data structure for storing the results, mapping from - // param value to a mapping from task id to result value - Map> resultMap = new TreeMap>(); - - // loop over all the results obtained from OpenML - for (Evaluation e : results.getEvaluations()) { - if (!resultMap.containsKey(e.getTask_id())) { - resultMap.put(e.getTask_id(), new TreeMap()); - } - resultMap.get(e.getTask_id()).put(e.getSetup_id(), e.getValue()); - } - - // initialize the csv writer and the header - BufferedWriter bw = new BufferedWriter(new FileWriter(resultsFile)); - bw.write("\"Task id\""); - for (int setupId : setupIds) { - bw.write("\t\"" + formatSetupid(setupId) + "\""); - } - - for (int taskId : taskIds) { - assertTrue("results doens't contain task " + taskId, resultMap.containsKey(taskId)); - bw.write("\n" + taskId); - for (int setupId : setupIds) { - if (resultMap.get(taskId).containsKey(setupId)) { - bw.write("\t" + resultMap.get(taskId).get(setupId)); - } else { - System.err.println("Warning: task " + taskId + " does not contain setup " + setupId); - bw.write("\t0.0"); - } - } - } - - bw.close(); - - /* now the file can be plotted with a GNUplot script like: - * - * set style data boxplot - * set xtics rotate by -45 - * sub(s) = system(sprintf("echo \"%s\" | sed 's/@/ /g'", s)) - * header = system("head -1 'results.csv'") - * set for [i=1:words(header)] xtics (sub(word(header, i)) i) - * plot for [i=2:40] 'results.csv' using (i):i lt -1 lc rgb "blue" pointsize 0.2 notitle - */ - } - - public static void flowsOnTask(OpenmlConnector openml, int taskId, String evaluationMeasure, File resultsFile, - int resultLimit) throws Exception { - // Data structure that maps from flow id to a list of results (on the - // evalutionMeasure) - Map> flowidResult = new TreeMap>(); - // Data structure that maps from result (on the evalutionMeasure) to - // flowId - Map> resultFlowid = new TreeMap>(Collections.reverseOrder()); - // Data structure to keep track of flow ids already used - TreeSet usedFlowids = new TreeSet(); - - // bookkeeping to prepare the call; the function requires a list of - // taskIds - List taskIds = new ArrayList(); - taskIds.add(taskId); - - // obtains the results from OpenML - EvaluationList results = openml.evaluationList(taskIds, null, evaluationMeasure, evaluationLimit); - - // prepare our data structures: - for (Evaluation e : results.getEvaluations()) { - // obtain relevant information - Integer flowid = e.getFlow_id(); - Double result = e.getValue(); - - // initialize the keys in bookkeeping data structures: - if (!flowidResult.containsKey(flowid)) { - flowidResult.put(flowid, new ArrayList()); - } - if (!resultFlowid.containsKey(result)) { - resultFlowid.put(result, new ArrayList()); - } - - // fill the data structures - flowidResult.get(flowid).add(result); - resultFlowid.get(result).add(flowid); - } - - // initialize the csv writer - BufferedWriter bw = new BufferedWriter(new FileWriter(resultsFile)); - bw.write("\"rank\", \"id\", \"classifier\", \"result\"\n"); - - int rank = 1; - // loop over the results in order of the "best result" - outer: for (Iterator itt = resultFlowid.keySet().iterator(); itt.hasNext();) { - Double result = itt.next(); - // note that for each result, we can have multiple flows that scored - // that result. Loop over these - for (int flowid : resultFlowid.get(result)) { - // obtain the flow from OpenML (for the name in the key) - Flow flow = openml.flowGet(flowid); - - // function that only returns true for flows we are interested - // in - if (flowEligible(flow) && !usedFlowids.contains(flow.getId())) { - // this flow could have had multiple runs with different - // hyper-parameters, loop over these - for (Double score : flowidResult.get(flowid)) { - bw.write(rank + "\t" + flowid + "\t\"" + formatFlowname(flow) + "\"\t" + score + "\n"); - } - - // generate some output - System.out.println( - rank + ". " + formatFlowname(flow) + ": " + flowidResult.get(flowid).size() + " results. "); - - // keep track that we used this flow (and will not reuse it) - usedFlowids.add(flow.getId()); - - // update rank - rank += 1; - - // Break if the plot contains more than resultLimit lines - if (rank > resultLimit) { - break outer; - } - } - } - } - bw.close(); - // now the resulting csv can be plotted with a GNUplot command like - // "plot 'results.csv' using 1:4:xticlabels(3)" - } - - public static void hyperparameterEffect(OpenmlConnector openml, List taskIds, List setupIds, - String hyperparameter, String evaluationMeasure, File resultsFile) throws Exception { - // obtains all evaluations that comply to the three filters - EvaluationList results = openml.evaluationList(taskIds, setupIds, evaluationMeasure, evaluationLimit); - - // initialize data structure for storing the results, mapping from - // param value to a mapping from task id to result value - Map> hyperparameterEffect = new TreeMap>(); - - // for sanity checking: all setups need to have the same flow id - Integer flowId = null; - - // loop over all the results obtained from OpenML - for (Evaluation e : results.getEvaluations()) { - // we have a setup id -> use this to obtain the full setup object - SetupParameters setupDetails = openml.setupParameters(e.getSetup_id()); - // sanity checking - if (flowId == null) { - flowId = setupDetails.getFlow_id(); - } else { - if (!flowId.equals(setupDetails.getFlow_id())) { - throw new RuntimeException("Flow id of setups does not match. Expected: " + flowId + ", found: " + setupDetails.getFlow_id()); - } - } - - // use convenience function to convert hyperparameters object into hashmap - Map params = setupDetails.getParametersAsMap(); - // obtain the value of the hyperparameter we are interested in - Double hyperparameterValue = Double.parseDouble(params.get(hyperparameter).getValue()); - // and add this to our data structure - if (!hyperparameterEffect.containsKey(hyperparameterValue)) { - hyperparameterEffect.put(hyperparameterValue, new TreeMap()); - } - hyperparameterEffect.get(hyperparameterValue).put(e.getTask_id(), e.getValue()); - - } - - // initialize the csv writer and the header - BufferedWriter bw = new BufferedWriter(new FileWriter(resultsFile)); - bw.write("\"" + hyperparameter + "\""); - for (int taskId : taskIds) { bw.write("\t\"Task " + taskId + "\""); } - bw.write("\n"); - - // loops over the results and print to csv - for (Iterator itt = hyperparameterEffect.keySet().iterator(); itt.hasNext();) { - Double paramVal = itt.next(); - bw.append(""+paramVal); - for (int taskId : taskIds) { - if (hyperparameterEffect.get(paramVal).containsKey(taskId)) { - bw.append("\t" + hyperparameterEffect.get(paramVal).get(taskId)); - } else { - bw.append(","); - } - } - bw.append("\n"); - } - bw.close(); - - // now the file can be plotted with a GNUplot command like "plot for [i=2:10] "results.csv" using 1:i with lp title columnheader" - } - - public static String formatSetupid(int setupId) throws Exception { - SetupParameters sp = openml.setupParameters(setupId); - Flow f = openml.flowGet(sp.getFlow_id()); - - return f.getName(); - } - - public static String formatFlowname(Flow f) { - // this function provides a name mapping in case you want to display - // custom names - return f.getName().substring(5) + "(" + f.getVersion() + ")"; - } - - public static boolean flowEligible(Flow f) { - return f.getName().startsWith("weka.") && f.getComponent() == null; - } -} diff --git a/apiconnector/src/test/java/utils/TestCacheFunctions.java b/apiconnector/src/test/java/utils/TestCacheFunctions.java deleted file mode 100644 index ce6ed0c5..00000000 --- a/apiconnector/src/test/java/utils/TestCacheFunctions.java +++ /dev/null @@ -1,115 +0,0 @@ -package utils; - -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.net.URL; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.lang3.tuple.Pair; -import org.apache.http.client.HttpResponseException; -import org.junit.Ignore; -import org.junit.Test; -import org.openml.apiconnector.io.HttpCacheController; -import org.openml.apiconnector.io.OpenmlConnector; -import org.openml.apiconnector.settings.Settings; -import org.openml.apiconnector.xml.DataSetDescription; -import org.openml.apiconnector.xml.Task; - -import testbase.BaseTestFramework; - -public class TestCacheFunctions extends BaseTestFramework { - - private static void utilDatasetCache(OpenmlConnector openml, int did) throws Exception { - Settings.CACHE_ALLOWED = true; - Settings.CACHE_DIRECTORY = System.getProperty("user.home") + "/.openml_test/cache"; - - String[] suffix = { - "datasets/" + did + "/description.xml", - "datasets/" + did + "/features.xml", - "datasets/" + did + "/dataset.arff", - }; - // first remove potential cache files, to ensure that this procedure placed them - for (String s : suffix) { - File toRemove = HttpCacheController.getCacheLocation(new URL(openml.getApiUrl()), s); - assertTrue(toRemove.getAbsolutePath().startsWith(Settings.CACHE_DIRECTORY)); - if (toRemove.exists()) { - toRemove.delete(); - } - } - - DataSetDescription dsd = openml.dataGet(did); - openml.dataFeatures(did); - openml.datasetGet(dsd); - - for (String s : suffix) { - File checkExists = HttpCacheController.getCacheLocation(new URL(openml.getApiUrl()), s); - assertTrue(checkExists.exists()); - } - - // redo the calls, to check all cache operations work fine - openml.dataGet(did); - openml.dataFeatures(did); - openml.datasetGet(dsd); - } - - private static void utilTaskCache(OpenmlConnector openml, List> expected, int taskId) throws Exception { - Settings.CACHE_ALLOWED = true; - Settings.CACHE_DIRECTORY = System.getProperty("user.home") + "/.openml_test/cache"; - // first remove potential cache files, to ensure that this procedure placed them - for (Pair pair : expected) { - File toRemove = HttpCacheController.getCacheLocation(new URL(pair.getRight()), pair.getLeft()); - assertTrue(toRemove.getAbsolutePath().startsWith(Settings.CACHE_DIRECTORY)); - toRemove.delete(); - } - - Task task = openml.taskGet(taskId); - openml.taskSplitsGet(task); - - for (Pair pair : expected) { - File checkExists = HttpCacheController.getCacheLocation(new URL(pair.getRight()), pair.getLeft()); - assertTrue(checkExists.exists()); - } - - // redo the calls, to check all cache operations work fine - openml.taskGet(taskId); - openml.taskSplitsGet(task); - } - - @Test - public void testDatasetLive() throws Exception { - utilDatasetCache(client_read_live, 61); - } - - @Test - public void testDatasetTest() throws Exception { - utilDatasetCache(client_read_test, 5); - } - - @Test - public void testTaskLive() throws Exception { - List> expected = Arrays.asList( - Pair.of("tasks/59/task.xml", "https://www.openml.org/api/v1/task/59"), - Pair.of("tasks/59/datasplits.arff", "https://api.openml.org/api_splits/get/59/Task_59_splits.arff") - ); - utilTaskCache(client_read_live, expected, 59); - } - - @Test - public void testTaskTest() throws Exception { - List> expected = Arrays.asList( - Pair.of("tasks/115/task.xml", "https://test.openml.org/api/v1/task/115"), - Pair.of("tasks/115/datasplits.arff", "https://test.openml.org/api_splits/get/115/Task_115_splits.arff") - ); - utilTaskCache(client_read_test, expected, 115); - } - - @Test(expected=HttpResponseException.class) - public void testCacheRejectsOnError() throws Exception { - Integer illegalTaskId = 999999; - URL illigalSplits = new URL(url_test + "/api_splits/get/" + illegalTaskId + "/Task_" + illegalTaskId + "_splits.arff"); - String suffix = "tasks/" + illegalTaskId + "/datasplits.arff"; - HttpCacheController.getCachedFileFromUrl(illigalSplits, suffix); - } -}