Mercurial > hg > thermostat-ng
changeset 28:00540d33ce40 version 0.1.1
Changes to SwaggerCombine for web-gateway schema API
This patch modifies the thermostat-common module by upgrading the
functionality of SwaggerCombine and the YAML parser (which is NOT a full
YAML parser, but only the subset we use) to handle the use cases
about to be implemented in the web-gateway.
- better parsing of YAML arrays
- output and @atfile support in SwaggerCombine
- misc bug fixes
The version number is also ungraded to 0.1.1
Reviewed-by: sgehwolf
Review-thread: http://icedtea.classpath.org/pipermail/thermostat/2017-October/025364.html
| author | Simon Tooke <stooke@redhat.com> |
|---|---|
| date | Wed, 18 Oct 2017 10:28:09 -0400 |
| parents | a119766f44bc |
| children | dafb9de45e69 |
| files | common/pom.xml common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombine.java common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContext.java common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContextBuilder.java common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineMain.java common/src/main/java/com/redhat/thermostat/common/yaml/YamlToJson.java common/src/test/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContextTest.java common/src/test/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineTest.java pom.xml |
| diffstat | 9 files changed, 325 insertions(+), 64 deletions(-) [+] |
line wrap: on
line diff
--- a/common/pom.xml Thu Oct 12 13:01:38 2017 +0200 +++ b/common/pom.xml Wed Oct 18 10:28:09 2017 -0400 @@ -6,7 +6,7 @@ <groupId>com.redhat.thermostat</groupId> <artifactId>thermostat-common</artifactId> - <version>0.1.0</version> + <version>0.1.1</version> <packaging>bundle</packaging> <!-- required for uploading to Maven Central -->
--- a/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombine.java Thu Oct 12 13:01:38 2017 +0200 +++ b/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombine.java Wed Oct 18 10:28:09 2017 -0400 @@ -40,6 +40,9 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; import java.io.Writer; import java.util.List; import java.util.Map; @@ -68,13 +71,30 @@ public SwaggerCombine() { } + /** + * perform combine, writing output to a Writer + * @param ctx input context object + * @param out output writer, to which YAML or JSON is writen depending on the context + * @return object representing the combined context + * @throws IOException on i/o or syntax error + */ public JsonObject run(final SwaggerCombineContext ctx, Writer out) throws IOException { - JsonObject root = processAPIs(ctx); + JsonObject root = run(ctx); writeObject(root, out, ctx.getFmt() == SwaggerCombineContext.OutputFormat.YAML, ctx.isPretty()); return root; } + /** + * perform combine + * @param ctx input context object + * @return object representing the combined context + * @throws IOException on i/o or syntax error + */ + public JsonObject run(final SwaggerCombineContext ctx) throws IOException { + return processAPIs(ctx.getAPIList(), ctx.isQuiet(), ctx.isUseTemplate(), ctx.isLint()); + } + private void writeObject(JsonObject root, Writer out, boolean toYaml, boolean pretty) throws IOException { if (root != null) { @@ -86,24 +106,37 @@ } } - private JsonObject processAPIs(final SwaggerCombineContext ctx) throws IOException { - return processAPIs(ctx.getAPIList(), ctx.isQuiet(), ctx.isUseTemplate(), ctx.isLint()); - } - JsonObject processAPIs(final List<JsonObject> apiList, boolean quiet, boolean useTemplate, boolean doLint) throws IOException { + JsonObject processAPIs(final List<SwaggerCombineContext.MicroAPI> apiList, boolean quiet, boolean useTemplate, boolean doLint) throws IOException { JsonObject root = null; if (useTemplate) { - final File templateFile = new File(getClass().getResource(SWAGGER_TEMPLATE).getFile()); - root = processAPI(null, readSwaggerFile(templateFile), doLint); + root = processAPI(null, readSwaggerStream(getClass().getResourceAsStream(SWAGGER_TEMPLATE), false), doLint); } - for (JsonObject apiDef : apiList) { - // read in the Swagger YAML - if (!quiet) { - info("processing " + apiDef.get("")); + for (SwaggerCombineContext.MicroAPI apiArg : apiList) { + final JsonObject json; + switch (apiArg.getFormat()) { + case YAML_STRING: + final YamlToJson y2j = new YamlToJson(); + json = y2j.yamlToJsonObject(apiArg.getData()); + break; + case JSON_STRING: + final JsonParser parser = new JsonParser(); + json = parser.parse(new BufferedReader(new StringReader(apiArg.getData()))).getAsJsonObject(); + break; + case FILE: + // read in the Swagger YAML + if (!quiet) { + info("processing " + apiArg.getInFile()); + } + json = readSwaggerFile(apiArg.getInFile()); + break; + default: + // just to get rid of compiler warning + json = null; } - root = processAPI(root, apiDef, doLint); + root = processAPI(root, json, doLint); } return root; } @@ -259,7 +292,32 @@ } } - static JsonObject readSwaggerFile(File fin) throws IOException { + /** + * read a JSON or YAML stream into a JsonObject + * @param in input stream + * @param isYaml true if YAML format, false if JSON format + * @return a JsonObject representing the input stream contents + * @throws IOException if there was an error reading, or an error in format + */ + public static JsonObject readSwaggerStream(InputStream in, boolean isYaml) throws IOException { + final JsonObject json; + if (isYaml) { + final YamlToJson y2j = new YamlToJson(); + json = y2j.yamlToJsonObject(new BufferedReader(new InputStreamReader(in))); + } else { + final JsonParser parser = new JsonParser(); + json = parser.parse(new BufferedReader(new InputStreamReader(in))).getAsJsonObject(); + } + return json; + } + + /** + * read a JSON or YAML file into a JsonObject + * @param fin input file (must end in ".yaml" or ".json") + * @return a JsonObject representing the filecontents + * @throws IOException if there was an error reading, or an error in format + */ + public static JsonObject readSwaggerFile(File fin) throws IOException { final JsonObject json; if (fin.getName().endsWith(".yaml")) { final YamlToJson y2j = new YamlToJson();
--- a/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContext.java Thu Oct 12 13:01:38 2017 +0200 +++ b/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContext.java Wed Oct 18 10:28:09 2017 -0400 @@ -36,19 +36,14 @@ package com.redhat.thermostat.common.swaggercombine; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.redhat.thermostat.common.yaml.YamlToJson; - -import java.io.BufferedReader; import java.io.File; import java.io.IOException; -import java.io.StringReader; import java.util.ArrayList; import java.util.List; /** - * Utility class to build a SwaggerCombine object + * Utility class containing execution context for SwaggerCombine + * (not modified by SwaggerCombine) */ public class SwaggerCombineContext { @@ -60,17 +55,75 @@ private boolean useTemplate = false; private boolean lint = true; private boolean printUsage = false; - private final List<JsonObject> apiList = new ArrayList<>(); + private String outputFile= null; + private final List<MicroAPI> apiList = new ArrayList<>(); // potential extra args to allow for when building arg array private static final int EXTRA_ARG_COUNT = 5; + public static class MicroAPI { + + public enum InputFormat { FILE, YAML_STRING, JSON_STRING }; + + private final InputFormat fmt; + private final String data; + private final File infile; + + private MicroAPI(InputFormat fmt, String data) { + this.fmt = fmt; + this.data = data; + this.infile = null; + } + + private MicroAPI(InputFormat fmt, File in) { + this.fmt = fmt; + this.data = null; + this.infile = in; + } + + static MicroAPI createFromYaml(String yaml) { + return new MicroAPI(InputFormat.YAML_STRING, yaml); + } + + static MicroAPI createFromJson(String json) { + return new MicroAPI(InputFormat.JSON_STRING, json); + } + + static MicroAPI createFromFile(File fn) { + return new MicroAPI(InputFormat.FILE, fn); + } + + InputFormat getFormat() { + return fmt; + } + + String getData() { + return data; + } + + File getInFile() { + return infile; + } + } + public SwaggerCombineContext() { } + public String getOutputFile() { + return outputFile; + } + /** - * defint the output format + * set an output file (defaults to stdout) + * @param outputFile name of file (null or '-' for stdout) + */ + public void setOutputFile(String outputFile) { + this.outputFile = outputFile; + } + + /** + * define the output format * @param fmt one of OutputFormat.YAML | JSON or NONE * @return this */ @@ -127,10 +180,8 @@ * @throws IOException if an I/O error occurred */ public SwaggerCombineContext addMicroAPI(File fin) throws IOException { - final JsonObject json = SwaggerCombine.readSwaggerFile(fin); - if (json != null) { - this.getAPIList().add(json); - } + final MicroAPI api = MicroAPI.createFromFile(fin); + this.getAPIList().add(api); return this; } @@ -142,15 +193,9 @@ * @throws IOException if an I/O error occurred */ public SwaggerCombineContext addMicroAPI(String spec, boolean isYaml) throws IOException { - final JsonObject json; - if (isYaml) { - final YamlToJson y2j = new YamlToJson(); - json = y2j.yamlToJsonObject(spec); - } else { - final JsonParser parser = new JsonParser(); - json = parser.parse(new BufferedReader(new StringReader(spec))).getAsJsonObject(); - } - this.getAPIList().add(json); + + final MicroAPI api = isYaml ? MicroAPI.createFromYaml(spec) : MicroAPI.createFromJson(spec); + this.getAPIList().add(api); return this; } @@ -192,7 +237,7 @@ return printUsage; } - public List<JsonObject> getAPIList() { + public List<MicroAPI> getAPIList() { return apiList; }
--- a/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContextBuilder.java Thu Oct 12 13:01:38 2017 +0200 +++ b/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContextBuilder.java Wed Oct 18 10:28:09 2017 -0400 @@ -36,16 +36,33 @@ package com.redhat.thermostat.common.swaggercombine; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +/** + * build a context object from command line arguements + */ public class SwaggerCombineContextBuilder { + /** + * build a context object from command line arguements + * @param args input arg list + * @return new context object reflecting the input arguments + * @throws IOException if @file doesn't exist + */ public SwaggerCombineContext buildContext(String[] args) throws IOException { + final List<String> expandedArgs = expandArgs(args); + final SwaggerCombineContext ctx = new SwaggerCombineContext(); - for (final String arg : args) { - if ("--quiet".equals(arg)) { + for (final String arg : expandedArgs) { + if (arg == null) { + System.err.println("null argument: (ignored)"); + } else if ("--quiet".equals(arg)) { ctx.quiet(true); } else if ("--pretty".equals(arg)) { ctx.prettyPrint(true); @@ -53,6 +70,23 @@ ctx.produce(SwaggerCombineContext.OutputFormat.JSON); } else if ("--yaml".equals(arg)) { ctx.produce(SwaggerCombineContext.OutputFormat.YAML); + } else if (arg.startsWith("--output=")) { + final String outFn = arg.substring(arg.indexOf('=') + 1); + if ("-".equals(outFn)) { + ctx.setOutputFile(null); + } else { + ctx.setOutputFile(outFn); + } + if (ctx.getFmt() == SwaggerCombineContext.OutputFormat.NONE) { + if (outFn.endsWith(".yaml")) { + ctx.produce(SwaggerCombineContext.OutputFormat.YAML); + } else if (outFn.endsWith(".json")) { + ctx.produce(SwaggerCombineContext.OutputFormat.JSON); + } else { + System.err.println("unable to determine output format from output extension - assuming YAML"); + ctx.produce(SwaggerCombineContext.OutputFormat.YAML); + } + } } else if ("--help".equals(arg)) { ctx.usage(true); } else if ("--lint".equals(arg)) { @@ -69,4 +103,25 @@ return ctx; } + /** + * replace @filename with arguments from file (one line == one argument) + * @param inargs input arg list + * @return revices arglist with file contents inline + * @throws IOException if file does not exist or can't be read + */ + private List<String> expandArgs(String inargs[]) throws IOException { + List<String> xargs = new ArrayList<>(inargs.length * 2); + for (String arg : inargs) { + if (arg == null || arg.length() < 2 || !arg.startsWith("@")) { + xargs.add(arg); + } else { + BufferedReader in = new BufferedReader(new FileReader(arg.substring(1))); + for (String line = in.readLine(); line != null; line = in.readLine()) { + xargs.add(line); + } + } + } + return xargs; + } + }
--- a/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineMain.java Thu Oct 12 13:01:38 2017 +0200 +++ b/common/src/main/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineMain.java Wed Oct 18 10:28:09 2017 -0400 @@ -37,6 +37,7 @@ package com.redhat.thermostat.common.swaggercombine; import java.io.BufferedWriter; +import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; @@ -54,10 +55,14 @@ final SwaggerCombine combine = new SwaggerCombine(); final SwaggerCombineContext ctx = new SwaggerCombineContextBuilder().buildContext(args); if (ctx.printUsage()) { - System.err.println("SwaggerCombine [--yaml] [--json [--pretty]] [--lint] [--use-template] [--quiet] [--help] infile1 [infile2] ..."); - } else { + System.err.println("SwaggerCombine [--yaml] [--json [--pretty]] [--lint] [--use-template] [--quiet] [--help] [@argfile] infile1 [infile2] ..."); + } else if (ctx.getOutputFile() == null) { final Writer out = new BufferedWriter(new PrintWriter(System.out)); combine.run(ctx, out); + } else { + final Writer out = new BufferedWriter(new FileWriter(ctx.getOutputFile())); + combine.run(ctx, out); + out.close(); } } catch (IOException e) { e.printStackTrace();
--- a/common/src/main/java/com/redhat/thermostat/common/yaml/YamlToJson.java Thu Oct 12 13:01:38 2017 +0200 +++ b/common/src/main/java/com/redhat/thermostat/common/yaml/YamlToJson.java Wed Oct 18 10:28:09 2017 -0400 @@ -50,8 +50,6 @@ import java.io.IOException; import java.io.Reader; import java.io.StringReader; -import java.io.StringWriter; -import java.io.Writer; import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -99,7 +97,7 @@ * @throws IOException if a syntax error, or an I/O error */ public JsonObject yamlToJsonObject(final Reader in) throws IOException { - try (final PushableReader pr = new PushableReader(in instanceof BufferedReader ? (BufferedReader)in : new BufferedReader(in));) { + try (final PushableReader pr = new PushableReader(in instanceof BufferedReader ? (BufferedReader)in : new BufferedReader(in))) { final JsonObject obj = new JsonObject(); yamlToJson(pr, obj, 0); pr.close(); @@ -123,7 +121,11 @@ if (line.value != null && !line.value.isEmpty()) { if (">-".equals(line.value)) { //the next few lines are the value for the current line - child = makeUnquotedPrimitive(absorb(in, Integer.max(currentIndent, line.indent))); + child = makeUnquotedPrimitive(absorb(in, Math.max(currentIndent, line.indent))); + } else if (line.value.startsWith("[")) { + // [ array elements ] (may be on separate lines + in.push(line.value); + child = absorbArray(in, line.indent); } else { JsonElement ce = makeUnquotedPrimitive(line.value); if (parent.isJsonArray()) { @@ -183,9 +185,53 @@ } } + // allows for arrays on one line: foo: [one, two, threee] + private JsonArray absorbArray(PushableReader in, int currentIndent) throws IOException { + // create an array and read primitives into it until we hit ']' + final JsonArray array = new JsonArray(); + in.resetTokenizer(); + // skip past '[' + in.nextTokenOnLine(); + String t = in.nextTokenOnLine(); + while (t != null) { + if ("]".equals(t)) { + break; + } else { + array.add(makeUnquotedPrimitive(t)); + } + t = in.nextTokenOnLine(); + } + if (!"]".equals(t)) { + // read following lines as name value pairs + for (String s = in.readLine(); s != null; s = in.readLine()) { + YamlLine line = new YamlLine(s); + if (line.indent == 0 && line.value == null && line.name == null) { + // blsnk or unparseable line + continue; + } + if (line.indent < currentIndent) { + in.push(s); + return array; + } + if (line.value != null) { + if (line.name != null) { + JsonObject obj = new JsonObject(); + obj.add(makeUnquotedString(line.name), makeUnquotedPrimitive(line.value)); + array.add(obj); + } else if ("]".equals(line.value)) { + break; + } else { + array.add(makeUnquotedPrimitive(t)); + } + } + } + } + return array; + } + private static String makeUnquotedString(final String s) { String ret = s; - if (s.length() >= 2) { + if (s.length() > 2) { if ((s.charAt(0) == '\'') && (s.charAt(s.length() - 1) == '\'') || (s.charAt(0) == '"') && (s.charAt(s.length() - 1) == '"')) { ret = s.substring(1, s.length() - 1); @@ -211,7 +257,7 @@ try { return new JsonPrimitive(new Long(uqString)); } catch (NumberFormatException ignored) { - ; // fall through to return String + // fall through to return String } } @@ -296,6 +342,9 @@ final Stack<String> stack = new Stack<>(); final BufferedReader in; + Matcher matcher; + static final Pattern pattern = Pattern.compile("(\\[|\\]|[\"'\\w]+)"); + PushableReader(BufferedReader in) { this.in = in; } @@ -312,10 +361,48 @@ stack.push(s); } + void resetTokenizer() { + matcher = null; + } + @Override public void close() throws IOException { in.close(); } + + String nextToken() throws IOException { + // tokens are separated by space or comma + if (matcher == null) { + final String line = readLine(); + matcher = (line != null) ? pattern.matcher(line) : null; + } + if (matcher != null) { + if (matcher.find()) { + return matcher.group(); + } else { + // ran out of tokens - try next line + matcher = null; + return nextToken(); + } + } + return null; + } + + String nextTokenOnLine() throws IOException { + // tokens are separated by space or comma + if (matcher == null) { + final String line = readLine(); + matcher = (line != null) ? pattern.matcher(line) : null; + } + if (matcher != null) { + if (matcher.find()) { + return matcher.group(); + } else { + matcher = null; + } + } + return null; + } } /**
--- a/common/src/test/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContextTest.java Thu Oct 12 13:01:38 2017 +0200 +++ b/common/src/test/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineContextTest.java Wed Oct 18 10:28:09 2017 -0400 @@ -36,17 +36,14 @@ package com.redhat.thermostat.common.swaggercombine; -import com.redhat.thermostat.common.swaggercombine.SwaggerCombineContext; import org.junit.Test; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class SwaggerCombineContextTest { @@ -99,20 +96,12 @@ assertEquals(SwaggerCombineContext.OutputFormat.YAML, ctx.getFmt()); } - @Test(expected = IOException.class) - public void testInvalidFiles() throws IOException { - final SwaggerCombineContext ctx = new SwaggerCombineContext(); - assertEquals(0, ctx.getAPIList().size()); - ctx.addMicroAPI(new File(SOME_INVALID_FILE)); - fail(); - } - - @Test(expected = FileNotFoundException.class) + @Test public void testMissingFiles() throws IOException { final SwaggerCombineContext ctx = new SwaggerCombineContext(); assertEquals(0, ctx.getAPIList().size()); ctx.addMicroAPI(new File(SOME_MISSING_FILE)); - fail(); + assertEquals(1, ctx.getAPIList().size()); } @Test
--- a/common/src/test/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineTest.java Thu Oct 12 13:01:38 2017 +0200 +++ b/common/src/test/java/com/redhat/thermostat/common/swaggercombine/SwaggerCombineTest.java Wed Oct 18 10:28:09 2017 -0400 @@ -81,11 +81,10 @@ assertTrue(buf.toString().contains("key offset differs")); } - private List<JsonObject> toJsonList(File[] files) throws IOException { - List<JsonObject> list = new ArrayList<>(files.length); + private List<SwaggerCombineContext.MicroAPI> toJsonList(File[] files) throws IOException { + List<SwaggerCombineContext.MicroAPI> list = new ArrayList<>(files.length); for (File fn : files) { - JsonObject spec = SwaggerCombine.readSwaggerFile(fn); - list.add(spec); + list.add(SwaggerCombineContext.MicroAPI.createFromFile(fn)); } return list; }
--- a/pom.xml Thu Oct 12 13:01:38 2017 +0200 +++ b/pom.xml Wed Oct 18 10:28:09 2017 -0400 @@ -66,6 +66,29 @@ </properties> <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <!-- + 3.6+ is JDK 9 compatible. See: + https://cwiki.apache.org/confluence/display/MAVEN/Java+9+-+Jigsaw + --> + <version>3.6.1</version> + </plugin> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <version>1.4.0</version> + </plugin> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-scr-plugin</artifactId> + <version>1.21.0</version> + </plugin> + </plugins> + </pluginManagement> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId>