BenchmarkRunner.java - thrift-protobuf-compare - Comparing varius aspects of Serialization libraries on the JVM platform

Source path: svn/ trunk/ tpc/ src/ serializers/ BenchmarkRunner.java

‹r126

r222

package serializers;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;

import serializers.avro.AvroGenericSerializer;
import serializers.avro.specific.AvroSpecificSerializer;
import serializers.kryo.KryoOptimizedSerializer;
import serializers.kryo.KryoSerializer;

public class BenchmarkRunner
{
  public final static int ITERATIONS = 2000;
  public final static int TRIALS = 20;

  /**
   * Number of milliseconds to warm up for each operation type for each serializer. Let's
   * start with 3 seconds.
   */
  final static long WARMUP_MSECS = 3000;

  @SuppressWarnings("unchecked")
  private Set<ObjectSerializer> _serializers = new LinkedHashSet<ObjectSerializer>();

  public static void main(String... args) throws Exception
  {
    BenchmarkRunner runner = new BenchmarkRunner();

    // binary codecs first
    runner.addObjectSerializer(new AvroGenericSerializer());
    runner.addObjectSerializer(new AvroSpecificSerializer());
    runner.addObjectSerializer(new ActiveMQProtobufSerializer());
    runner.addObjectSerializer(new ProtobufSerializer());
    runner.addObjectSerializer(new ThriftSerializer());
    runner.addObjectSerializer(new HessianSerializer());
    runner.addObjectSerializer(new KryoSerializer());
    runner.addObjectSerializer(new KryoOptimizedSerializer());
    runner.addObjectSerializer(new MessagePackSerializer());

    // then language default serializers
    runner.addObjectSerializer(new JavaSerializer());

    runner.addObjectSerializer(new JavaExtSerializer());
    runner.addObjectSerializer(new ScalaSerializer());

    // then Json
    runner.addObjectSerializer(new JsonSerializer());
    runner.addObjectSerializer(new JsonDataBindingSerializer());

    runner.addObjectSerializer(new JsonMarshallerSerializer());

    runner.addObjectSerializer(new ProtostuffJsonSerializer());
    runner.addObjectSerializer(new ProtostuffNumericJsonSerializer());
    // this is pretty slow; so slow that it's almost not worth keeping but:
    runner.addObjectSerializer(new GsonSerializer());

    // then xml via stax, textual and binary
    runner.addObjectSerializer(new StaxSerializer("stax/woodstox",
                                                  new com.ctc.wstx.stax.WstxInputFactory(),
                                                  new com.ctc.wstx.stax.WstxOutputFactory()));
    runner.addObjectSerializer(new StaxSerializer("stax/aalto",
                                                  new com.fasterxml.aalto.stax.InputFactoryImpl(),
                                                  new com.fasterxml.aalto.stax.OutputFactoryImpl()));

    runner.addObjectSerializer(new StaxSerializer("binaryxml/FI",
                                                  new com.sun.xml.fastinfoset.stax.factory.StAXInputFactory(),
                                                  new com.sun.xml.fastinfoset.stax.factory.StAXOutputFactory()));

    // No point in running all 4 variants: let's just use fastest one:
    //runner.addObjectSerializer(new XStreamSerializer("xstream (xpp)", false, null, null));
    //runner.addObjectSerializer(new XStreamSerializer("xstream (xpp with conv)", true, null, null));
    //runner.addObjectSerializer(new XStreamSerializer("xstream (stax)", false, new com.ctc.wstx.stax.WstxInputFactory(), new com.ctc.wstx.stax.WstxOutputFactory()));
    runner.addObjectSerializer(new XStreamSerializer("xstream (stax with conv)",
                                                     true,
                                                     new com.ctc.wstx.stax.WstxInputFactory(),
                                                     new com.ctc.wstx.stax.WstxOutputFactory()));
    runner.addObjectSerializer(new JavolutionXMLFormatSerializer());

    runner.addObjectSerializer(new SbinarySerializer());

    System.out.println("Starting");
    runner.start();
  }

  @SuppressWarnings("unchecked")
  private void addObjectSerializer(ObjectSerializer serializer)
  {
    _serializers.add(serializer);
  }

  private <T> double createObjects(ObjectSerializer<T> serializer, int iterations) throws Exception
  {
    long start = System.nanoTime();
    for (int i = 0; i < iterations; i++)
    {
      serializer.create();
    }
    return iterationTime(System.nanoTime() - start, iterations);
  }

  private double iterationTime(long delta, int iterations)
  {
    return (double) delta / (double) (iterations);
  }

  private <T> double serializeDifferentObjects(ObjectSerializer<T> serializer, int iterations) throws Exception
  {
    long start = System.nanoTime();
    for (int i = 0; i < iterations; i++)
    {
      T obj = serializer.create();
      serializer.serialize(obj);
    }
    return iterationTime(System.nanoTime()-start, iterations);
  }
  
  
  private <T> double serializeSameObject(ObjectSerializer<T> serializer, int iterations) throws Exception
  {
    // let's reuse same instance to reduce overhead
    T obj = serializer.create();
    long delta = 0;
    for (int i = 0; i < iterations; i++)
    {
      long start = System.nanoTime();
      serializer.serialize(obj);
      delta += System.nanoTime() - start;
      if (i % 1000 == 0)
        doGc();
    }
    return iterationTime(delta, iterations);
  }  

  private <T> double deserializeNoFieldAccess(ObjectSerializer<T> serializer, int iterations) throws Exception
  {
    byte[] array = serializer.serialize(serializer.create());
    long start = System.nanoTime();
    T result = null;
    for (int i = 0; i < iterations; i++)
    {
      result = serializer.deserialize(array);
    }
    return iterationTime(System.nanoTime()-start, iterations);
  }

  private <T> double deserializeAndCheckAllFields(CheckingObjectSerializer<T> serializer, int iterations) throws Exception
  {
    byte[] array = serializer.serialize(serializer.create());
    long delta = 0;
    for (int i = 0; i < iterations; i++)
    {
      long start = System.nanoTime();
      T obj = serializer.deserialize(array);
      serializer.checkAllFields(obj);
      delta += System.nanoTime() - start;
    }
    return iterationTime(delta, iterations);
  }  
  
  private <T> double deserializeAndCheckMediaField(CheckingObjectSerializer<T> serializer, int iterations) throws Exception
  {
    byte[] array = serializer.serialize(serializer.create());
    long delta = 0;
    for (int i = 0; i < iterations; i++)
    {
      long start = System.nanoTime();
      T obj = serializer.deserialize(array);
      serializer.checkMediaField(obj);
      delta += System.nanoTime() - start;
    }
    return iterationTime(delta, iterations);
  }   
  
  /**
   * JVM is not required to honor GC requests, but adding bit of sleep around request is
   * most likely to give it a chance to do it.
   */
  private void doGc()
  {
      try {
          Thread.sleep(50L);
      } catch (InterruptedException ie) { }
      System.gc();
      try { // longer sleep afterwards (not needed by GC, but may help with scheduling)
          Thread.sleep(200L);
      } catch (InterruptedException ie) { }
  }

  enum measurements
  {
    timeCreate, timeSerializeDifferentObjects, timeSerializeSameObject, timeDeserializeNoFieldAccess, timeDeserializeAndCheckMediaField, timeDeserializeAndCheckAllFields, totalTime, length
  }

  @SuppressWarnings("unchecked")
  private void start() throws Exception
  {
    System.out.printf("%-24s, %15s, %15s, %15s, %15s, %15s, %15s, %15s, %10s\n",
                      " ",
                      "Object create",
                      "Serialize",
                      "/w Same Object",
                      "Deserialize",
                      "and Check Media",
                      "and Check All",
                      "Total Time",
                      "Serialized Size");
    EnumMap<measurements, Map<String, Double>> values = new EnumMap<measurements, Map<String, Double>>(measurements.class);
    for (measurements m : measurements.values())
      values.put(m, new HashMap<String, Double>());

    for (ObjectSerializer serializer : _serializers)
    {
      /*
       * Should only warm things for the serializer that we test next: HotSpot JIT will
       * otherwise spent most of its time optimizing slower ones... Use
       * -XX:CompileThreshold=1 to hint the JIT to start immediately
       *
       * Actually: 1 is often not a good value -- threshold is the number
       * of samples needed to trigger inlining, and there's no point in
       * inlining everything. Default value is in thousands, so lowering
       * it to, say, 1000 is usually better.
       */
      warmCreation(serializer);
      doGc();
      double timeCreate = Double.MAX_VALUE;
      // do more iteration for object creation because of its short time
      for (int i = 0; i < TRIALS; i++)
        timeCreate = Math.min(timeCreate, createObjects(serializer, ITERATIONS * 100));

      warmSerialization(serializer);

      // actually: let's verify serializer actually works now:
      checkCorrectness(serializer);

      doGc();
      double timeSerializeDifferentObjects = Double.MAX_VALUE;
      for (int i = 0; i < TRIALS; i++)
        timeSerializeDifferentObjects = Math.min(timeSerializeDifferentObjects, serializeDifferentObjects(serializer, ITERATIONS));

      doGc();
      double timeSerializeSameObject = Double.MAX_VALUE;
      for (int i = 0; i < TRIALS; i++)
          timeSerializeSameObject = Math.min(timeSerializeSameObject, serializeSameObject(serializer, ITERATIONS));
      
      warmDeserialization(serializer);
      
      doGc();
      double timeDeserializeNoFieldAccess = Double.MAX_VALUE;
      for (int i = 0; i < TRIALS; i++)
        timeDeserializeNoFieldAccess = Math.min(timeDeserializeNoFieldAccess, deserializeNoFieldAccess(serializer, ITERATIONS));

      double timeDeserializeAndCheckAllFields = timeDeserializeNoFieldAccess;
      double timeDeserializeAndCheckMediaField = timeDeserializeNoFieldAccess;

      double totalTime = timeSerializeDifferentObjects + timeDeserializeNoFieldAccess;

      if( serializer instanceof CheckingObjectSerializer) {
          CheckingObjectSerializer checkingSerializer = (CheckingObjectSerializer)serializer;

          timeDeserializeAndCheckMediaField = Double.MAX_VALUE;
          doGc();
          for (int i = 0; i < TRIALS; i++)
              timeDeserializeAndCheckMediaField = Math.min(timeDeserializeAndCheckMediaField, deserializeAndCheckMediaField(checkingSerializer, ITERATIONS));

          timeDeserializeAndCheckAllFields = Double.MAX_VALUE;
          doGc();
          for (int i = 0; i < TRIALS; i++) 
              timeDeserializeAndCheckAllFields = Math.min(timeDeserializeAndCheckAllFields, deserializeAndCheckAllFields(checkingSerializer, ITERATIONS));

          totalTime = timeSerializeDifferentObjects + timeDeserializeAndCheckAllFields;
      }
            
      byte[] array = serializer.serialize(serializer.create());
      System.out.printf("%-24s, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %10d\n",
                        serializer.getName(),
                        timeCreate,
                        timeSerializeDifferentObjects,
                        timeSerializeSameObject,
                        timeDeserializeNoFieldAccess,
                        timeDeserializeAndCheckMediaField,
                        timeDeserializeAndCheckAllFields,
                        totalTime,
                        array.length);

     	addValue(values, serializer.getName(), timeCreate, timeSerializeDifferentObjects, timeSerializeSameObject, 
             timeDeserializeNoFieldAccess, timeDeserializeAndCheckMediaField, timeDeserializeAndCheckAllFields, totalTime, array.length);
    }
    printImages(values);
  }

    /**
     * Method that tries to validate correctness of serializer, using
     * round-trip (construct, serializer, deserialize; compare objects
     * after steps 1 and 3).
     * Currently only done for StdMediaDeserializer...
     */
    private void checkCorrectness(ObjectSerializer serializer)
        throws Exception
    {
        Object input = serializer.create();
        byte[] array = serializer.serialize(input);
        Object output = serializer.deserialize(array);

        if (!input.equals(output)) {
            /* Should throw an exception; but for now (that we have a few
             * failures) let's just whine...
             */
            String msg = "serializer '"+serializer.getName()+"' failed round-trip test (ser+deser produces Object different from input), input="+input+", output="+output;
            //throw new Exception("Error: "+msg);
            System.err.println("WARN: "+msg);
        }
    }

  private void printImages(EnumMap<measurements, Map<String, Double>> values)
  {
    for (measurements m : values.keySet()) {
     Map<String, Double> map = values.get(m);
     ArrayList<Entry> list = new ArrayList(map.entrySet());
     Collections.sort(list, new Comparator<Entry>() {
            public int compare (Entry o1, Entry o2) {
                double diff = (Double)o1.getValue() - (Double)o2.getValue();
                return diff > 0 ? 1 : (diff < 0 ? -1 : 0);
            }
     });
     LinkedHashMap<String, Double> sortedMap = new LinkedHashMap<String, Double>();
     for (Entry<String, Double> entry : list) {
         if( !entry.getValue().isNaN() ) {
             sortedMap.put(entry.getKey(), entry.getValue());
         }
     }
      if (!sortedMap.isEmpty()) printImage(sortedMap, m);
    }
  }

  private void printImage(Map<String, Double> map, measurements m)
  {
    StringBuilder valSb = new StringBuilder();
    String names = "";
    double max = Double.MIN_NORMAL;
    for (Entry<String, Double> entry : map.entrySet())
    {
      valSb.append(entry.getValue()).append(',');
      max = Math.max(max, entry.getValue());
      names = entry.getKey() + '|' + names;
    }

    int height = Math.min(30+map.size()*20, 430); 
    double scale = max * 1.1;
    System.out.println("<img src='http://chart.apis.google.com/chart?chtt="
        + m.name()
        + "&chf=c||lg||0||FFFFFF||1||76A4FB||0|bg||s||EFEFEF&chs=689x"+height+"&chd=t:"
        + valSb.toString().substring(0, valSb.length() - 1)
        + "&chds=0,"+ scale
        + "&chxt=y"
        + "&chxl=0:|" + names.substring(0, names.length() - 1)
        + "&chm=N *f*,000000,0,-1,10&lklk&chdlp=t&chco=660000|660033|660066|660099|6600CC|6600FF|663300|663333|663366|663399|6633CC|6633FF|666600|666633|666666&cht=bhg&chbh=10&nonsense=aaa.png'/>");

  }

  private void addValue(EnumMap<measurements, Map<String, Double>> values,
                        String name,
                        double timeCreate,
                        double timeSerializeDifferentObjects,
                        double timeSerializeSameObject,
                        double timeDeserializeNoFieldAccess,
                        double timeDeserializeAndCheckMediaField, 
                        double timeDeserializeAndCheckAllFields,
                        double totalTime,
                        double length)
  {
    // Omit some charts for serializers that are extremely slow.
    if (!name.equals("json/google-gson") && !name.equals("scala")) {
	    values.get(measurements.timeSerializeDifferentObjects).put(name, timeSerializeDifferentObjects);
	    values.get(measurements.timeSerializeSameObject).put(name, timeSerializeSameObject);
	    values.get(measurements.timeDeserializeNoFieldAccess).put(name, timeDeserializeNoFieldAccess);
	    values.get(measurements.timeDeserializeAndCheckMediaField).put(name, timeDeserializeAndCheckMediaField);
	    values.get(measurements.timeDeserializeAndCheckAllFields).put(name, timeDeserializeAndCheckAllFields);
	    values.get(measurements.totalTime).put(name, totalTime);
    }
    values.get(measurements.length).put(name, length);
    values.get(measurements.timeCreate).put(name, timeCreate);
  }

  private <T> void warmCreation(ObjectSerializer<T> serializer) throws Exception
  {
    // Instead of fixed counts, let's try to prime by running for N seconds
    long endTime = System.currentTimeMillis() + WARMUP_MSECS;
    do
    {
      createObjects(serializer, 1);
    }
    while (System.currentTimeMillis() < endTime);
  }

  private <T> void warmSerialization(ObjectSerializer<T> serializer) throws Exception
  {
    // Instead of fixed counts, let's try to prime by running for N seconds
    long endTime = System.currentTimeMillis() + WARMUP_MSECS;
    do
    {
      serializeDifferentObjects(serializer, 1);
    }
    while (System.currentTimeMillis() < endTime);
  }

  private <T> void warmDeserialization(ObjectSerializer<T> serializer) throws Exception
  {
    // Instead of fixed counts, let's try to prime by running for N seconds
    long endTime = System.currentTimeMillis() + WARMUP_MSECS;
    do
    {
      deserializeNoFieldAccess(serializer, 1);
    }
    while (System.currentTimeMillis() < endTime);
  }
}

Show details Hide details

Change log

r221 by pascallouisperez on Jul 23, 2011 Diff

Adding MessagePack to the serializers
being tested. Resolves  issue 27 .

Go to:

Older revisions

r126 by bl...@orcaware.com on Jan 24, 2010 Diff

Use svn_apply_autoprops to apply a
standard set of Subversion properties
to
all files.  The 'svn:eol-style' is set
to 'native' and 'svn:keywords' is set
...

r124 by nathan.sweet on Jan 21, 2010 Diff

Put in the deserialization time for
the deserialization + check fields
time for tests that don't need to
check fields.
Fixed bug that prevented charts from
...

r117 by nathan.sweet on Jan 10, 2010 Diff

Updated JsonMarshallerSerializer to
work with asm-3.2 so there isn't a
conflict with Kryo.

All revisions of this file

File info

Size: 17134 bytes, 424 lines

View raw file

File properties

svn:eol-style: native
svn:keywords: HeadURL Id LastChangedBy LastChangedD...Revision