move and update Viz

.gitignore

@@ -8,3 +8,4 @@
 *.csv
 /config/config.properties
 /logs/viz.log
+/viz/*.html

build.gradle

@@ -47,7 +47,8 @@ task runViz(type: JavaExec) {
     dependsOn classes
     systemProperty 'logback.configurationFile', 'src/main/resources/logback-viz.xml'
     classpath sourceSets.main.runtimeClasspath
-    main = 'Viz'
+    main = 'ats.viz.Viz'
+    args 'logs/viz.log', 'viz/viz.html'
 }
 
 testlogger {

src/main/java/Viz.java

@@ -1,34 +0,0 @@
-import java.util.StringJoiner;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Viz converts visualization log data into a graphical
- * representation.
- */
-public class Viz {
-	public static final String LOG_NAME = "viz-data";
-    static final Logger log = LoggerFactory.getLogger("Viz");
-    static final Logger viz = LoggerFactory.getLogger(LOG_NAME);
-
-
-    /**
-     * Run the converter.
-     */
-    public static void main(String[] args) {
-
-    }
-
-    /**
-     * Convert event data to our preferred format.  This format can be
-     * placed into a log, which we can later read back in.
-     */
-    public static String eventString(Object... objects) {
-        StringJoiner sj = new StringJoiner(", ");
-        for (Object o : objects) {
-            sj.add(o != null ? o.toString() : "null");
-        }
-        return sj.toString();
-    }
-}

src/main/java/ats/viz/Viz.java

@@ -0,0 +1,305 @@
+package ats.viz;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.StringJoiner;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.thymeleaf.TemplateEngine;
+import org.thymeleaf.context.Context;
+import org.thymeleaf.templatemode.TemplateMode;
+import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver;
+
+/**
+ * Viz converts visualization log data into a graphical
+ * representation.
+ */
+public class Viz {
+    static final Logger log = LoggerFactory.getLogger("Viz");
+    static final Gson gson = new GsonBuilder().create();
+    static final DateTimeFormatter formatter = DateTimeFormat.forPattern("E MMM d, yyyy, HH:mm:ss.SSS");
+    static int canvasWidth = 1000;
+    static int canvasHeight = 1000;
+    static int chartWidth = canvasWidth - 50;
+    static int chartHeight = 50;
+    DateTime startTime;
+    DateTime endTime;
+    BigDecimal midMin;
+    BigDecimal midMax;
+    List<Map<String, Object>> ticks = new ArrayList<>();
+    Map<String, List<Map<String, Object>>> bars = new HashMap<>();
+    // Map<String, List<Map<String, Object>>> events = new HashMap<>();
+    Map<String,EventSequence> events = new HashMap<>();
+
+
+    public class EventSequence {
+        String name;
+        Map<DateTime,List<Map<String,Object>>> events = new HashMap<>();
+        List<Float> points = new ArrayList<>();
+
+        EventSequence(String n) { name = n; }
+
+        void addEvent(Map<String,Object> e) {
+            DateTime time = (DateTime)e.get("time");
+
+            // {"price":12,"name":"open","instrument":"EUR_USD","id":"order_0001","units":10,"type":"event"}
+
+            log.debug("add event with time {}", time);
+            List<Map<String,Object>> myevents = events.get(time);
+
+            if (myevents == null) {
+                myevents = new ArrayList<>();
+                events.put(time, myevents);
+            }
+
+            myevents.add(e);
+        }
+
+        public List<Float> timePoints() {
+            long minTime = startTime.getMillis();
+            long maxTime = endTime.getMillis();
+            long timespan = maxTime - minTime;
+            float xscale = timespan / chartWidth;
+
+            List<Float> points = new ArrayList<>();
+            log.debug("{} events", events.size());
+            for (Map.Entry<DateTime, List<Map<String,Object>>> entry : events.entrySet()) {
+                log.debug("event: {} / {}", entry.getKey(), entry.getValue());
+                long tickTime = entry.getKey().getMillis();
+
+                float x = (tickTime - minTime) / xscale;
+                log.debug("adding point {}", x);
+                points.add(x);
+
+                // TODO: check for multiple
+            }
+
+            return points;
+        }
+    }
+
+    /**
+     * Run the converter.
+     */
+    public static void main(String[] args) throws FileNotFoundException, IOException {
+        if (args.length < 2) {
+            log.error("Not enough arguments.  Need log file and html file args.");
+            System.exit(1);
+        }
+
+        new Viz().process(args[0], args[1]);
+    }
+
+    /**
+     * Read a visualization log file and write out a graphical
+     * representation.
+     */
+    private void process(String logfile, String htmlfile) throws FileNotFoundException, IOException {
+        loadFile(logfile);
+        writeSummary();
+        writeHTML(htmlfile);
+    }
+
+    private void loadFile(String logfile) throws FileNotFoundException, IOException {
+        try (BufferedReader br = new BufferedReader(new FileReader(logfile))) {
+            DateTime currentTime = null;
+            for (String line; (line = br.readLine()) != null;) {
+                Map<String, String> tmp = gson.fromJson(line, new TypeToken<Map<String, String>>() {}.getType());
+                Map<String, Object> map = clean(tmp);
+
+                // make sure everything has a timestamp.  assume
+                // things that don't happen at the most recent time.
+                if (map.containsKey("time"))
+                    currentTime = (DateTime)map.get("time");
+
+                if (!map.containsKey("time"))
+                    map.put("time", currentTime);
+
+                if (!map.containsKey("type")) {
+                    log.warn("Skipping untyped object: {}", map);
+                    continue;
+                } else if ("tick".equals(map.get("type"))) {
+                    updateWindow((DateTime)map.get("time"));
+                    updateMidBounds((BigDecimal)map.get("mid"));
+                    ticks.add(map);
+                } else if ("bar".equals(map.get("type"))) {
+                    String name = (String) map.get("name");
+                    List<Map<String, Object>> list = bars.get(name);
+                    if (list == null) {
+                        list = new ArrayList<>();
+                        bars.put(name, list);
+                    }
+                    list.add(map);
+                } else if ("event".equals(map.get("type"))) {
+                    String name = (String)map.get("name");
+                    EventSequence seq = events.get(name);
+                    if (seq == null) {
+                        seq = new EventSequence(name);
+                        events.put(name, seq);
+                    }
+                    seq.addEvent(map);
+                } else {
+                    log.warn("Unknown object type: {}", map);
+                }
+            }
+        }
+    }
+
+    /**
+     * Convert from string types to more useful ones.
+     */
+    private Map<String, Object> clean(Map<String, String> map) {
+        Map<String, Object> out = new HashMap<>();
+
+        Pattern numberPattern = Pattern.compile("^[\\d.]+$");
+
+        for (Map.Entry<String, String> entry : map.entrySet()) {
+            String key = entry.getKey();
+            String val = entry.getValue();
+
+            Matcher m = numberPattern.matcher(val);
+            if (m.find()) {
+                // convert numbers
+                out.put(key, new BigDecimal(val));
+            } else {
+                out.put(key, val);
+            }
+        }
+
+        // convert dates
+        if (out.containsKey("time")) {
+            out.put("time", DateTime.parse((String) out.get("time")));
+        }
+
+        return out;
+    }
+
+    /**
+     * Update start and end times for the given time.
+     */
+    private void updateWindow(DateTime time) {
+        if (startTime == null) {
+            startTime = time;
+        } else if (time.isBefore(startTime)) {
+            startTime = time;
+        }
+
+        if (endTime == null) {
+            endTime = time;
+        } else if (time.isAfter(endTime)) {
+            endTime = time;
+        }
+    }
+
+    /**
+     * Update high and low values for tick price value
+     */
+    private void updateMidBounds(BigDecimal mid) {
+        if (midMin == null) {
+            midMin = mid;
+        } else if (mid.compareTo(midMin) < 0) {
+            midMin = mid;
+        }
+
+        if (midMax == null) {
+            midMax = mid;
+        } else if (mid.compareTo(midMax) > 0) {
+            midMax = mid;
+        }
+    }
+
+    private TemplateEngine templateEngine;
+
+    private void writeHTML(String htmlfile) {
+        ClassLoaderTemplateResolver templateResolver =
+            new ClassLoaderTemplateResolver(getClass().getClassLoader());
+        templateResolver.setTemplateMode(TemplateMode.HTML);
+        templateResolver.setPrefix("/viz/templates/");
+        templateResolver.setSuffix(".html");
+
+        templateEngine = new TemplateEngine();
+        templateEngine.setTemplateResolver(templateResolver);
+
+        Map<String, Object> vars = new HashMap<>();
+        vars.put("canvasWidth", canvasWidth);
+        vars.put("canvasHeight", canvasHeight);
+        vars.put("chartWidth", chartWidth);
+        vars.put("chartHeight", chartHeight);
+        vars.put("start", formatter.print(startTime));
+        vars.put("end", formatter.print(endTime));
+        vars.put("bars", bars);
+        vars.put("events", events);
+        vars.put("tickpoints", makeTickPoints(chartWidth, chartHeight));
+
+        Context context = new Context(Locale.getDefault(), vars);
+
+        try (FileWriter writer = new FileWriter(htmlfile)) {
+            templateEngine.process("test", context, writer);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        log.info("Wrote html to {}", htmlfile);
+    }
+
+    private String makeTickPoints(int chartWidth, int chartHeight)
+    {
+        long minTime = startTime.getMillis();
+        long maxTime = endTime.getMillis();
+        long timespan = maxTime - minTime;
+
+        float xscale = timespan / chartWidth;
+
+        float minVal = midMin.floatValue();
+        float maxVal = midMax.floatValue();
+        float valspan = maxVal - minVal;
+
+        float yscale = valspan / chartHeight;
+
+        // List<float[]> points = new ArrayList<>();
+
+        StringJoiner pointstr = new StringJoiner(" ");
+        for (Map<String, Object> tick : ticks) {
+            // {"name":"C","mid":1.052685,"time":"2017-01-02T00:00:00.803Z","type":"tick"}
+            long tickTime = ((DateTime)tick.get("time")).getMillis();
+            float x = (tickTime - minTime) / xscale;
+
+            float val = ((BigDecimal)tick.get("mid")).floatValue();
+            float y = (val - minVal) / yscale;
+
+            //points.add(new float[] {x, y});
+            pointstr.add(String.format("%.2f,%.2f", x, y));
+        }
+
+		return pointstr.toString();
+	}
+
+	/**
+     * Write a short list of what we've processed.
+     */
+    private void writeSummary() {
+        log.info("{} ticks, values from {} to {}, from {} to {}",
+                 ticks.size(), midMin, midMin, startTime, endTime);
+        log.info("{} bars", bars.size());
+        log.info("{} events", events.size());
+    }
+}

src/main/java/ats/viz/VizLog.java

@@ -0,0 +1,62 @@
+package ats.viz;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.espertech.esper.client.EventBean;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Viz converts visualization log data into a graphical
+ * representation.
+ */
+public class VizLog {
+    private static final Logger log = LoggerFactory.getLogger("VizLog");
+    private static final Logger viz = LoggerFactory.getLogger("viz-data");
+    private static final DateTimeFormatter dateFormatter = ISODateTimeFormat.dateTime();
+    private static final Gson gson = new GsonBuilder().create();
+
+
+	/**
+     * Write an event to the visualization log.
+     */
+    public static void log(EventBean[] data) {
+        viz.info(gson.toJson(data[0].get("event")));
+    }
+
+    /**
+     * Convert event data to our preferred format.  This format can be
+     * placed into a log, which we can later read back in.
+     */
+    public static Map<String,Object> eventString(String type, String tag, Object o) {
+        Map<String,Object> map = new HashMap<>();
+
+        map.put("type", type);
+        map.put("name", tag);
+
+        Map<String,Object> omap = new HashMap<>();
+        if (o instanceof Map) {
+            omap = (Map)o;
+        } else if (o instanceof VizMappable) {
+            omap = ((VizMappable)o).toVizPropertyMap();
+        } else {
+            log.warn("Can't convert to viz properties: {}", o);
+        }
+        map.putAll(omap);
+
+        // clean up some values
+        if (map.containsKey("time") && map.get("time") instanceof DateTime) {
+            DateTime dt = (DateTime)map.get("time");
+            map.put("time", dateFormatter.print(dt));
+        }
+            
+        return map;
+    }
+}

src/main/java/ats/viz/VizMappable.java

@@ -0,0 +1,16 @@
+package ats.viz;
+
+import java.util.Map;
+
+/**
+ * VizMappable marks a class as being able to be used in the
+ * visualization log.
+ */
+public interface VizMappable {
+
+    /**
+     * Return a map of property names to object values of the
+     * properties that should be included in the visualization log.
+     */
+    public Map<String,Object> toVizPropertyMap();
+}

src/main/resources/viz/templates/test.html

@@ -0,0 +1,93 @@
+<!DOCTYPE html>
+<html xmlns:th="http://www.thymeleaf.org">
+  <head>
+    <title>Visualization</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <link href="https://fonts.googleapis.com/css?family=Roboto:300,700" rel="stylesheet">
+    <style>
+     body {
+         font-family: 'Roboto', sans-serif;
+         margin: 20px;
+     }
+     .chartline {
+         stroke: black;
+         stroke-width: 2;
+     }
+     .tickline {
+         stroke: hsl(240, 100%, 50%);
+         stroke-width: 1;
+         fill: none;
+         vector-effect: non-scaling-stroke;
+     }
+     .eventlabel {
+         fill: hsl(0, 0%, 50%);
+         font-size: .8em;
+         text-anchor: end;
+         dominant-baseline: middle;
+     }
+     .eventline {
+         stroke: hsl(0, 0%, 70%);
+         stroke-width: 1;
+         fill: none;
+     }
+     .event {
+         stroke: hsl(300, 50%, 20%);
+         stroke-width: 1;
+         fill: hsl(300, 50%, 70%);
+     }
+     .secondEvent {
+         stroke: hsl(240, 50%, 50%);
+         stroke-width: 1;
+         fill: none;
+     }
+    </style>
+  </head>
+  <body>
+
+    <h3>From [[${start}]] to [[${end}]]</h3>
+
+    <svg width="100%" th:viewBox="|0 0 ${canvasWidth} ${canvasHeight}|" version="1.1"
+         xmlns="http://www.w3.org/2000/svg"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         th:with="chartstartx=50">
+      <defs>
+        <circle id="event" r="4" class="event" />
+        <g id="multiEvent">
+          <circle cy="3" r="4" class="event" />
+          <circle r="4" class="event" />
+        </g>
+      </defs>
+
+      <g id="bars"
+         th:with="starty=0,height=50,margin=25"
+         th:each="data,iterstat : ${bars}">
+        <g class="bar"
+           th:transform="|translate(${chartstartx}, ${starty + (height + margin) * iterstat.index})|">
+          <line class="chartline" x1="0" x2="0" y1="0" th:y2="${height}" />
+          <polyline class="tickline" th:points="${tickpoints}" />
+        </g>
+      </g>
+
+      <g id="events"
+         th:with="starty=80,height=10,margin=10"
+         th:each="eventseq,iterstat : ${events}">
+        <g class="eventgroup"
+           th:transform="|translate(0, ${starty + (height + margin) * iterstat.index})|">
+          <!-- label -->
+          <text class="eventlabel" th:x="${chartstartx - 10}" y="0" th:text="${eventseq.key}"></text>
+          <g id="eventpointswrapper"
+             th:transform="|translate(${chartstartx}, 0)|">
+            <!-- line -->
+            <line class="eventline" x1="0" th:x2="${chartWidth}" y1="0" y2="0"/>
+            <!-- points -->
+            <g id="eventpoints"
+               th:each="point : ${eventseq.value.timePoints()}">
+              <use th:x="${point}" xlink:href="#event" />
+            </g>
+          </g>
+        </g>
+      </g>
+
+    </svg>
+  </body>
+</html>

viz/README

@@ -0,0 +1,3 @@
+Visualization output files are kept in this directory.
+
+This file is so git can track the otherwise empty dir.