diff --git a/.gitignore b/.gitignore index 8a0305b..6879b45 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ *.csv /config/config.properties /logs/viz.log +/viz/*.html \ No newline at end of file diff --git a/build.gradle b/build.gradle index 1b6ed17..124e797 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,8 @@ task runViz(type: JavaExec) { dependsOn classes systemProperty 'logback.configurationFile', 'src/main/resources/logback-viz.xml' classpath sourceSets.main.runtimeClasspath - main = 'Viz' + main = 'ats.viz.Viz' + args 'logs/viz.log', 'viz/viz.html' } testlogger { diff --git a/src/main/java/Viz.java b/src/main/java/Viz.java deleted file mode 100644 index d665a5b..0000000 --- a/src/main/java/Viz.java +++ /dev/null @@ -1,34 +0,0 @@ -import java.util.StringJoiner; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Viz converts visualization log data into a graphical - * representation. - */ -public class Viz { - public static final String LOG_NAME = "viz-data"; - static final Logger log = LoggerFactory.getLogger("Viz"); - static final Logger viz = LoggerFactory.getLogger(LOG_NAME); - - - /** - * Run the converter. - */ - public static void main(String[] args) { - - } - - /** - * Convert event data to our preferred format. This format can be - * placed into a log, which we can later read back in. - */ - public static String eventString(Object... objects) { - StringJoiner sj = new StringJoiner(", "); - for (Object o : objects) { - sj.add(o != null ? o.toString() : "null"); - } - return sj.toString(); - } -} diff --git a/src/main/java/ats/viz/Viz.java b/src/main/java/ats/viz/Viz.java new file mode 100644 index 0000000..6af6adb --- /dev/null +++ b/src/main/java/ats/viz/Viz.java @@ -0,0 +1,305 @@ +package ats.viz; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.StringJoiner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.thymeleaf.TemplateEngine; +import org.thymeleaf.context.Context; +import org.thymeleaf.templatemode.TemplateMode; +import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver; + +/** + * Viz converts visualization log data into a graphical + * representation. + */ +public class Viz { + static final Logger log = LoggerFactory.getLogger("Viz"); + static final Gson gson = new GsonBuilder().create(); + static final DateTimeFormatter formatter = DateTimeFormat.forPattern("E MMM d, yyyy, HH:mm:ss.SSS"); + static int canvasWidth = 1000; + static int canvasHeight = 1000; + static int chartWidth = canvasWidth - 50; + static int chartHeight = 50; + DateTime startTime; + DateTime endTime; + BigDecimal midMin; + BigDecimal midMax; + List> ticks = new ArrayList<>(); + Map>> bars = new HashMap<>(); + // Map>> events = new HashMap<>(); + Map events = new HashMap<>(); + + + public class EventSequence { + String name; + Map>> events = new HashMap<>(); + List points = new ArrayList<>(); + + EventSequence(String n) { name = n; } + + void addEvent(Map e) { + DateTime time = (DateTime)e.get("time"); + + // {"price":12,"name":"open","instrument":"EUR_USD","id":"order_0001","units":10,"type":"event"} + + log.debug("add event with time {}", time); + List> myevents = events.get(time); + + if (myevents == null) { + myevents = new ArrayList<>(); + events.put(time, myevents); + } + + myevents.add(e); + } + + public List timePoints() { + long minTime = startTime.getMillis(); + long maxTime = endTime.getMillis(); + long timespan = maxTime - minTime; + float xscale = timespan / chartWidth; + + List points = new ArrayList<>(); + log.debug("{} events", events.size()); + for (Map.Entry>> entry : events.entrySet()) { + log.debug("event: {} / {}", entry.getKey(), entry.getValue()); + long tickTime = entry.getKey().getMillis(); + + float x = (tickTime - minTime) / xscale; + log.debug("adding point {}", x); + points.add(x); + + // TODO: check for multiple + } + + return points; + } + } + + /** + * Run the converter. + */ + public static void main(String[] args) throws FileNotFoundException, IOException { + if (args.length < 2) { + log.error("Not enough arguments. Need log file and html file args."); + System.exit(1); + } + + new Viz().process(args[0], args[1]); + } + + /** + * Read a visualization log file and write out a graphical + * representation. + */ + private void process(String logfile, String htmlfile) throws FileNotFoundException, IOException { + loadFile(logfile); + writeSummary(); + writeHTML(htmlfile); + } + + private void loadFile(String logfile) throws FileNotFoundException, IOException { + try (BufferedReader br = new BufferedReader(new FileReader(logfile))) { + DateTime currentTime = null; + for (String line; (line = br.readLine()) != null;) { + Map tmp = gson.fromJson(line, new TypeToken>() {}.getType()); + Map map = clean(tmp); + + // make sure everything has a timestamp. assume + // things that don't happen at the most recent time. + if (map.containsKey("time")) + currentTime = (DateTime)map.get("time"); + + if (!map.containsKey("time")) + map.put("time", currentTime); + + if (!map.containsKey("type")) { + log.warn("Skipping untyped object: {}", map); + continue; + } else if ("tick".equals(map.get("type"))) { + updateWindow((DateTime)map.get("time")); + updateMidBounds((BigDecimal)map.get("mid")); + ticks.add(map); + } else if ("bar".equals(map.get("type"))) { + String name = (String) map.get("name"); + List> list = bars.get(name); + if (list == null) { + list = new ArrayList<>(); + bars.put(name, list); + } + list.add(map); + } else if ("event".equals(map.get("type"))) { + String name = (String)map.get("name"); + EventSequence seq = events.get(name); + if (seq == null) { + seq = new EventSequence(name); + events.put(name, seq); + } + seq.addEvent(map); + } else { + log.warn("Unknown object type: {}", map); + } + } + } + } + + /** + * Convert from string types to more useful ones. + */ + private Map clean(Map map) { + Map out = new HashMap<>(); + + Pattern numberPattern = Pattern.compile("^[\\d.]+$"); + + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + + Matcher m = numberPattern.matcher(val); + if (m.find()) { + // convert numbers + out.put(key, new BigDecimal(val)); + } else { + out.put(key, val); + } + } + + // convert dates + if (out.containsKey("time")) { + out.put("time", DateTime.parse((String) out.get("time"))); + } + + return out; + } + + /** + * Update start and end times for the given time. + */ + private void updateWindow(DateTime time) { + if (startTime == null) { + startTime = time; + } else if (time.isBefore(startTime)) { + startTime = time; + } + + if (endTime == null) { + endTime = time; + } else if (time.isAfter(endTime)) { + endTime = time; + } + } + + /** + * Update high and low values for tick price value + */ + private void updateMidBounds(BigDecimal mid) { + if (midMin == null) { + midMin = mid; + } else if (mid.compareTo(midMin) < 0) { + midMin = mid; + } + + if (midMax == null) { + midMax = mid; + } else if (mid.compareTo(midMax) > 0) { + midMax = mid; + } + } + + private TemplateEngine templateEngine; + + private void writeHTML(String htmlfile) { + ClassLoaderTemplateResolver templateResolver = + new ClassLoaderTemplateResolver(getClass().getClassLoader()); + templateResolver.setTemplateMode(TemplateMode.HTML); + templateResolver.setPrefix("/viz/templates/"); + templateResolver.setSuffix(".html"); + + templateEngine = new TemplateEngine(); + templateEngine.setTemplateResolver(templateResolver); + + Map vars = new HashMap<>(); + vars.put("canvasWidth", canvasWidth); + vars.put("canvasHeight", canvasHeight); + vars.put("chartWidth", chartWidth); + vars.put("chartHeight", chartHeight); + vars.put("start", formatter.print(startTime)); + vars.put("end", formatter.print(endTime)); + vars.put("bars", bars); + vars.put("events", events); + vars.put("tickpoints", makeTickPoints(chartWidth, chartHeight)); + + Context context = new Context(Locale.getDefault(), vars); + + try (FileWriter writer = new FileWriter(htmlfile)) { + templateEngine.process("test", context, writer); + } catch (Exception e) { + e.printStackTrace(); + } + + log.info("Wrote html to {}", htmlfile); + } + + private String makeTickPoints(int chartWidth, int chartHeight) + { + long minTime = startTime.getMillis(); + long maxTime = endTime.getMillis(); + long timespan = maxTime - minTime; + + float xscale = timespan / chartWidth; + + float minVal = midMin.floatValue(); + float maxVal = midMax.floatValue(); + float valspan = maxVal - minVal; + + float yscale = valspan / chartHeight; + + // List points = new ArrayList<>(); + + StringJoiner pointstr = new StringJoiner(" "); + for (Map tick : ticks) { + // {"name":"C","mid":1.052685,"time":"2017-01-02T00:00:00.803Z","type":"tick"} + long tickTime = ((DateTime)tick.get("time")).getMillis(); + float x = (tickTime - minTime) / xscale; + + float val = ((BigDecimal)tick.get("mid")).floatValue(); + float y = (val - minVal) / yscale; + + //points.add(new float[] {x, y}); + pointstr.add(String.format("%.2f,%.2f", x, y)); + } + + return pointstr.toString(); + } + + /** + * Write a short list of what we've processed. + */ + private void writeSummary() { + log.info("{} ticks, values from {} to {}, from {} to {}", + ticks.size(), midMin, midMin, startTime, endTime); + log.info("{} bars", bars.size()); + log.info("{} events", events.size()); + } +} diff --git a/src/main/java/ats/viz/VizLog.java b/src/main/java/ats/viz/VizLog.java new file mode 100644 index 0000000..9c053b3 --- /dev/null +++ b/src/main/java/ats/viz/VizLog.java @@ -0,0 +1,62 @@ +package ats.viz; + +import java.util.HashMap; +import java.util.Map; + +import com.espertech.esper.client.EventBean; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Viz converts visualization log data into a graphical + * representation. + */ +public class VizLog { + private static final Logger log = LoggerFactory.getLogger("VizLog"); + private static final Logger viz = LoggerFactory.getLogger("viz-data"); + private static final DateTimeFormatter dateFormatter = ISODateTimeFormat.dateTime(); + private static final Gson gson = new GsonBuilder().create(); + + + /** + * Write an event to the visualization log. + */ + public static void log(EventBean[] data) { + viz.info(gson.toJson(data[0].get("event"))); + } + + /** + * Convert event data to our preferred format. This format can be + * placed into a log, which we can later read back in. + */ + public static Map eventString(String type, String tag, Object o) { + Map map = new HashMap<>(); + + map.put("type", type); + map.put("name", tag); + + Map omap = new HashMap<>(); + if (o instanceof Map) { + omap = (Map)o; + } else if (o instanceof VizMappable) { + omap = ((VizMappable)o).toVizPropertyMap(); + } else { + log.warn("Can't convert to viz properties: {}", o); + } + map.putAll(omap); + + // clean up some values + if (map.containsKey("time") && map.get("time") instanceof DateTime) { + DateTime dt = (DateTime)map.get("time"); + map.put("time", dateFormatter.print(dt)); + } + + return map; + } +} diff --git a/src/main/java/ats/viz/VizMappable.java b/src/main/java/ats/viz/VizMappable.java new file mode 100644 index 0000000..ac2462d --- /dev/null +++ b/src/main/java/ats/viz/VizMappable.java @@ -0,0 +1,16 @@ +package ats.viz; + +import java.util.Map; + +/** + * VizMappable marks a class as being able to be used in the + * visualization log. + */ +public interface VizMappable { + + /** + * Return a map of property names to object values of the + * properties that should be included in the visualization log. + */ + public Map toVizPropertyMap(); +} diff --git a/src/main/resources/viz/templates/test.html b/src/main/resources/viz/templates/test.html new file mode 100644 index 0000000..307fda9 --- /dev/null +++ b/src/main/resources/viz/templates/test.html @@ -0,0 +1,93 @@ + + + + Visualization + + + + + + +

From [[${start}]] to [[${end}]]

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/viz/README b/viz/README new file mode 100644 index 0000000..9d7d72a --- /dev/null +++ b/viz/README @@ -0,0 +1,3 @@ +Visualization output files are kept in this directory. + +This file is so git can track the otherwise empty dir.