001/*
002 * Units of Measurement TCK
003 * Copyright © 2005-2019, Jean-Marie Dautelle, Werner Keil, Otavio Santana.
004 *
005 * All rights reserved.
006 *
007 * Redistribution and use in source and binary forms, with or without modification,
008 * are permitted provided that the following conditions are met:
009 *
010 * 1. Redistributions of source code must retain the above copyright notice,
011 *    this list of conditions and the following disclaimer.
012 *
013 * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions
014 *    and the following disclaimer in the documentation and/or other materials provided with the distribution.
015 *
016 * 3. Neither the name of JSR-385 nor the names of its contributors may be used to endorse or promote products
017 *    derived from this software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
020 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
021 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
022 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
023 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
026 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
028 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package tech.units.tck;
031
032import static tech.units.tck.util.TestUtils.SYS_PROPERTY_OUTPUT_DIR;
033import static tech.units.tck.util.TestUtils.SYS_PROPERTY_PROFILE;
034import static tech.units.tck.util.TestUtils.SYS_PROPERTY_REPORT_FILE;
035import static tech.units.tck.util.TestUtils.SYS_PROPERTY_VERBOSE;
036
037import java.io.File;
038import java.io.FileWriter;
039import java.io.IOException;
040import java.io.InputStream;
041import java.io.OutputStream;
042import java.io.PrintWriter;
043import java.io.StringWriter;
044import java.lang.reflect.Method;
045import java.util.ArrayList;
046import java.util.Arrays;
047import java.util.Collections;
048import java.util.HashSet;
049import java.util.List;
050import java.util.Set;
051
052import javax.lang.model.SourceVersion;
053import javax.tools.Tool;
054
055import org.testng.ITestResult;
056import org.testng.TestListenerAdapter;
057import org.testng.TestNG;
058import org.testng.annotations.Test;
059import org.testng.reporters.VerboseReporter;
060import org.testng.xml.XmlClass;
061import org.testng.xml.XmlSuite;
062import org.testng.xml.XmlTest;
063
064import tech.units.tck.tests.FundamentalTypesTest;
065import tech.units.tck.tests.format.QuantityFormatTest;
066import tech.units.tck.tests.format.UnitFormatTest;
067import tech.units.tck.tests.quantity.QuantityInterfaceTest;
068import tech.units.tck.tests.quantity.QuantityTypesTest;
069import tech.units.tck.tests.spi.ObtainingQuantiesTest;
070import tech.units.tck.tests.spi.ObtainingUnitsTest;
071import tech.units.tck.tests.spi.QuantityFactoryTest;
072import tech.units.tck.tests.spi.ServiceProviderTest;
073import tech.units.tck.tests.spi.ServicesTest;
074import tech.units.tck.tests.spi.SystemOfUnitsTest;
075import tech.units.tck.tests.unit.PrefixInterfaceTest;
076import tech.units.tck.tests.unit.UnitConversionTest;
077import tech.units.tck.tests.unit.UnitDimensionTest;
078import tech.units.tck.tests.unit.UnitInterfaceTest;
079import tech.units.tck.util.TestGroups.Profile;
080import tech.uom.lib.common.function.Versioned;
081
082/**
083 * Main class for executing the JSR 385 TCK.
084 * 
085 * @author <a href="mailto:[email protected]">Werner Keil</a>
086 * @version 2.0, July 7, 2019
087 * @since 1.0
088 */
089public class TCKRunner extends XmlSuite implements Tool, Versioned<String> {
090
091    /**
092     * 
093     */
094    //private static final long serialVersionUID = 3189431432291353154L;
095    private static final String TCK_VERSION = "2.0.0";
096    public static final String SPEC_ID = "JSR 385";
097    public static final String SPEC_VERSION = "2.0.0";
098    private final Profile profile;
099
100    public TCKRunner() {
101        setName(SPEC_ID + " - TCK " + TCK_VERSION);
102        final XmlTest test = new XmlTest(this);
103        profile = Profile.valueOf((System.getProperty(SYS_PROPERTY_PROFILE, 
104                        Profile.FULL.name()).toUpperCase()));
105        for (String group : profile.getGroups()) {
106            test.addIncludedGroup(group);
107        }
108        test.setName("TCK/Test Setup");
109        final List<XmlClass> classes = new ArrayList<>();
110        classes.add(new XmlClass(TCKSetup.class));
111        classes.add(new XmlClass(FundamentalTypesTest.class));
112        classes.add(new XmlClass(UnitInterfaceTest.class));
113        classes.add(new XmlClass(UnitConversionTest.class));
114        classes.add(new XmlClass(PrefixInterfaceTest.class));
115        classes.add(new XmlClass(UnitDimensionTest.class));
116        classes.add(new XmlClass(QuantityInterfaceTest.class));
117        classes.add(new XmlClass(QuantityTypesTest.class));
118        classes.add(new XmlClass(UnitFormatTest.class));
119        classes.add(new XmlClass(QuantityFormatTest.class));
120        classes.add(new XmlClass(QuantityFactoryTest.class));
121        classes.add(new XmlClass(SystemOfUnitsTest.class));
122        classes.add(new XmlClass(ServiceProviderTest.class));
123        classes.add(new XmlClass(ServicesTest.class));
124        classes.add(new XmlClass(ObtainingUnitsTest.class));
125        classes.add(new XmlClass(ObtainingQuantiesTest.class));
126        test.setXmlClasses(classes);
127    }
128
129    /**
130     * Main method to start the TCK. Optional arguments are:
131     * <ul>
132     * <li>-Dtech.units.tck.profile for defining the profile for TestNG groups (default: full).</li>
133     * <li>-Dtech.units.tck.outputDir for defining the output directory TestNG uses (default:
134     * ./target/tck-output).</li>
135     * <li>-Dtech.units.tck.verbose=true to enable TestNG verbose mode.</li>
136     * <li>-Dtech.units.tck.reportFile=targetFile.txt for defining the TCK result summary report
137     * target file (default: ./target/tck-results.txt).</li>
138     * </ul>
139     * 
140     * @param args Optional arguments to control TCK execution
141     */
142    @SuppressWarnings("deprecation")
143        @Override
144    public int run(InputStream in, OutputStream out, OutputStream err, String... args) {
145        System.out.println("-- " + SPEC_ID + " TCK started --");
146        System.out.println("Profile: " + profile.getDescription());
147        final List<XmlSuite> suites = new ArrayList<>();
148        suites.add(new TCKRunner());
149        final TestNG tng = new TestNG();
150        tng.setXmlSuites(suites);
151        String outDir = System.getProperty(SYS_PROPERTY_OUTPUT_DIR, "./target/tck-output");
152        tng.setOutputDirectory(outDir);
153        String verbose = System.getProperty(SYS_PROPERTY_VERBOSE);
154        if ("true".equalsIgnoreCase(verbose)) {
155            tng.addListener(new VerboseReporter("[VerboseUoM] "));
156        }
157        String reportFile = System.getProperty(SYS_PROPERTY_REPORT_FILE, "./target/tck-results.txt");
158        final File file = new File(reportFile);
159        final Reporter rep = new Reporter(profile, file);
160        System.out.println("Writing to file " + file.getAbsolutePath() + " ...");
161        tng.addListener(rep);
162        tng.run();
163        rep.writeSummary();
164        System.out.println("-- " + SPEC_ID + " TCK finished --");
165        return 0;
166    }
167
168    @Override
169    public String getVersion() {
170        return TCK_VERSION;
171    }
172
173    @Override
174    public final Set<SourceVersion> getSourceVersions() {
175        return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(new SourceVersion[] {SourceVersion.RELEASE_5, SourceVersion.RELEASE_6, SourceVersion.RELEASE_7})));
176    }
177
178    public static final void main(String... args) {
179        if (args.length > 0 && "-version".equalsIgnoreCase(args[0])) {
180            showVersion();
181        } else { // (args.length > 0 && "-help".equalsIgnoreCase(args[0])) {
182            showHelp();
183        } /*
184           * else { final Tool runner = new TCKRunner(); runner.run(System.in, System.out,
185           * System.err, new String[]{TCKRunner.class.getName()}); }
186           */
187    }
188
189    private static void showHelp() {
190        final StringWriter consoleWriter = new StringWriter(1000);
191        consoleWriter.write("*****************************************************************************************\n");
192        consoleWriter.write("**** " + SPEC_ID + " - Units of Measurement, Technical Compatibility Kit, version " + TCK_VERSION + "\n");
193        consoleWriter.write("*****************************************************************************************\n\n");
194        consoleWriter.write("Usage:\n");
195        consoleWriter.write("To run the TCK, execute TestNG with Maven or a similar build tool.\n\n");
196        consoleWriter.write("E.g. by running \"mvn test\" with this POM.\n\n");
197        consoleWriter.write("You may use the following system properties to override the default behavior:\n");
198        consoleWriter.write("-D" + SYS_PROPERTY_PROFILE + "=<profile>" + " to select the desired profile from these available " + SPEC_ID + " profiles:\n");
199        for (Profile p : Profile.values()) {
200            consoleWriter.write("   " + p.name() + " - " + p.getDescription() + (p.isDefault() ? " (the default profile)\n" : "\n"));
201        }
202        consoleWriter.write("-D" + SYS_PROPERTY_OUTPUT_DIR + "=<directory> to set the output directory of your choice.\n");
203        consoleWriter.write("-D" + SYS_PROPERTY_REPORT_FILE + "=<file> to set the TCK result file directory of your choice.\n");
204        consoleWriter.write("-D" + SYS_PROPERTY_VERBOSE + "=true/false to toggle the TCK verbose option for additional test output. The default is \"false\"\n");
205        System.out.println(consoleWriter);
206    }
207
208    private static void showVersion() {
209        System.out.println(SPEC_ID + " - Units of Measurement, Technical Compatibility Kit, version \"" + TCK_VERSION + "\"\n");
210    }
211
212    public static final class Reporter extends TestListenerAdapter {
213        private int count = 0;
214        private int skipped = 0;
215        private int failed = 0;
216        private int success = 0;
217        private final StringWriter consoleWriter = new StringWriter(3000);
218        private FileWriter fileWriter;
219
220        public Reporter(Profile profile, File file) {
221            try {
222                if (!file.exists()) {
223                    file.createNewFile();
224                }
225                fileWriter = new FileWriter(file);
226                fileWriter.write("*****************************************************************************************\n");
227                fileWriter.write("**** " + SPEC_ID + " - Units of Measurement, Technical Compatibility Kit, version " + TCK_VERSION + "\n");
228                fileWriter.write("*****************************************************************************************\n\n");
229                fileWriter.write("Executed on " + new java.util.Date() + "\n");
230                fileWriter.write("Using " + profile.getDescription() + " profile\n\n");
231                // System.out:
232                consoleWriter.write("*****************************************************************************************\n");
233                consoleWriter.write("**** " + SPEC_ID + " - Units of Measurement, Technical Compatibility Kit, version " + TCK_VERSION + "\n");
234                consoleWriter.write("*****************************************************************************************\n\n");
235                consoleWriter.write("Executed on " + new java.util.Date() + "\n");
236                consoleWriter.write("Using " + profile.getDescription() + " profile\n\n");
237            } catch (IOException e) {
238                e.printStackTrace();
239                System.exit(-1);
240            }
241        }
242
243        @Override
244        public void onTestFailure(ITestResult tr) {
245            failed++;
246            count++;
247            String location = tr.getTestClass().getRealClass().getSimpleName() + '#' + tr.getMethod().getMethodName();
248            try {
249                Method realTestMethod = tr.getMethod().getConstructorOrMethod().getMethod();
250                Test testAnnot = realTestMethod.getAnnotation(Test.class);
251                if (testAnnot != null && testAnnot.description() != null && !testAnnot.description().isEmpty()) {
252                    if (tr.getThrowable() != null) {
253                        StringWriter sw = new StringWriter();
254                        PrintWriter w = new PrintWriter(sw);
255                        tr.getThrowable().printStackTrace(w);
256                        w.flush();
257                        log("[FAILED]  " + testAnnot.description() + "(" + location + "):\n" + sw.toString());
258                    } else {
259                        log("[FAILED]  " + testAnnot.description() + "(" + location + ")");
260                    }
261                } else {
262                    if (tr.getThrowable() != null) {
263                        StringWriter sw = new StringWriter();
264                        PrintWriter w = new PrintWriter(sw);
265                        tr.getThrowable().printStackTrace(w);
266                        w.flush();
267                        log("[FAILED]  " + location + ":\n" + sw.toString());
268                    } else {
269                        log("[FAILED]  " + location);
270                    }
271                }
272            } catch (IOException e) {
273                throw new IllegalStateException("IO Error", e);
274            }
275        }
276
277        @Override
278        public void onTestSkipped(ITestResult tr) {
279            skipped++;
280            count++;
281            String location = tr.getTestClass().getRealClass().getSimpleName() + '#' + tr.getMethod().getMethodName();
282            try {
283                Method realTestMethod = tr.getMethod().getConstructorOrMethod().getMethod();
284                Test specAssert = realTestMethod.getAnnotation(Test.class);
285                if (specAssert != null && specAssert.description() != null && !specAssert.description().isEmpty()) {
286                    log("[SKIPPED] " + specAssert.description() + "(" + location + ")");
287                } else {
288                    log("[SKIPPED] " + location);
289                }
290            } catch (IOException e) {
291                throw new IllegalStateException("IO Error", e);
292            }
293        }
294
295        @Override
296        public void onTestSuccess(ITestResult tr) {
297            success++;
298            count++;
299            String location = tr.getTestClass().getRealClass().getSimpleName() + '#' + tr.getMethod().getMethodName();
300            try {
301                Method realTestMethod = tr.getMethod().getConstructorOrMethod().getMethod();
302                Test specAssert = realTestMethod.getAnnotation(Test.class);
303                if (specAssert != null && specAssert.description() != null && !specAssert.description().isEmpty()) {
304                    log("[SUCCESS] " + specAssert.description() + "(" + location + ")");
305                } else {
306                    log("[SUCCESS] " + location);
307                }
308            } catch (IOException e) {
309                throw new IllegalStateException("IO Error", e);
310            }
311        }
312
313        private void log(String text) throws IOException {
314            fileWriter.write(text);
315            fileWriter.write('\n');
316            consoleWriter.write(text);
317            consoleWriter.write('\n');
318        }
319
320        public void writeSummary() {
321            try {
322                log("\n" + SPEC_ID + " TCK version " + TCK_VERSION + " Summary");
323                log("-------------------------------------------------------");
324                log("\nTOTAL TESTS EXECUTED : " + count);
325                log("TOTAL TESTS SKIPPED  : " + skipped);
326                log("TOTAL TESTS SUCCESS  : " + success);
327                log("TOTAL TESTS FAILED   : " + failed);
328                fileWriter.flush();
329                fileWriter.close();
330                consoleWriter.flush();
331                System.out.println();
332                System.out.println(consoleWriter);
333            } catch (IOException e) {
334                throw new IllegalStateException("IO Error", e);
335            }
336        }
337    }
338}