Decoding speedup

.containers/coatjava.Dockerfile

@@ -23,4 +23,4 @@ ARG REF_NAME=development
 RUN java --version && cd /opt && \
     git clone https://code.jlab.org/hallb/alert/coatjava.git && cd coatjava && \
     git fetch origin && git checkout ${REF_NAME} && ./build-coatjava.sh --quiet && \
-    ./install-clara /opt/clara
+    ./bin/install-clara /opt/clara

.github/workflows/ci.yml

@@ -88,7 +88,7 @@ jobs:
       - name: build
         run: |
           ./build-coatjava.sh --lfs --no-progress -T${{ env.nthreads }}
-          ./install-clara -b -c ./coatjava ./clara
+          ./bin/install-clara -b -c ./coatjava ./clara
       - name: tar # tarball to preserve permissions
         run: |
           tar czvf coatjava.tar.gz coatjava

.gitlab-ci.yml

@@ -70,9 +70,8 @@ build:
 download:
   stage: build
   script:
-#    - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./
     - git lfs install
-    - git clone https://code.jlab.org/hallb/clas12/validation-data
+    - git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data
     - cp validation-data/raw/rg-d/clas_018779.evio.00001 .
   artifacts:
     when: always
@@ -122,6 +121,19 @@ download:
     paths:
       - pages
 
+kpp:
+  stage: test
+  needs: [build]
+  dependencies: [build]
+  script:
+    - tar -xzf coatjava.tar.gz
+    - tar -xzf clara.tar.gz
+    - git config --global --add safe.directory $CI_PROJECT_DIR
+    - git lfs install
+    - git submodule update --init validation/advanced-tests/data
+    - cd validation/advanced-tests
+    - ./run-advanced-tests.sh
+
 eb:
   stage: test
   needs: [build]
@@ -148,7 +160,7 @@ decoder:
   dependencies: [build,download]
   script:
     - tar -xzf clara.tar.gz
-    - decoder -n 1000 -o clas_018779_00001.hipo clas_018779.evio.00001
+    - decoder4u -l FINE -n 10000 -o clas_018779_00001.hipo clas_018779.evio.00001
   artifacts:
     when: always
     expire_in: 1 day

bin/decoder4u

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+. `dirname $0`/../libexec/env.sh
+
+split_cli $@
+
+export MALLOC_ARENA_MAX=1
+
+java -Xmx2304m -Xms1280m -XX:+UseSerialGC ${jvm_options[@]} \
+    -cp ${COATJAVA_CLASSPATH:-''} \
+    org.jlab.detector.decode.CLASDecoder4U \
+    ${class_options[@]}
+

build-coatjava.sh

@@ -367,7 +367,6 @@ for pom in $(find common-tools -name pom.xml); do
 done
 echo "installed coatjava to: $prefix_dir"
 
-# install clara
-if $installClara; then ./install-clara -c $prefix_dir $clara_home; fi
+if $installClara; then ./bin/install-clara -c $prefix_dir $clara_home; fi
 
 echo "COATJAVA SUCCESSFULLY BUILT !"

common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Reader.java

@@ -0,0 +1,90 @@
+package org.jlab.io.clara;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Path;
+import org.jlab.clara.engine.EngineDataType;
+import org.jlab.clara.std.services.AbstractEventReaderService;
+import org.jlab.clara.std.services.EventReaderException;
+import org.jlab.coda.jevio.EvioException;
+import org.jlab.detector.decode.CLASDecoder4;
+import org.jlab.io.evio.EvioDataEvent;
+import org.jlab.io.evio.EvioSource;
+import org.jlab.jnp.hipo4.data.Event;
+import org.jlab.jnp.hipo4.io.HipoReader;
+import org.json.JSONObject;
+
+/**
+ *
+ * @author baltzell
+ */
+public class Clas12Reader extends AbstractEventReaderService<Object> {
+
+    boolean evio;
+    CLASDecoder4 decoder;
+    private long maxEvents;
+    private Double torus;
+    private Double solenoid;
+
+    @Override
+    protected Object createReader(Path path, JSONObject opts) throws EventReaderException {
+        if (path.toString().endsWith(".hipo")) {
+            evio = false;
+            HipoReader r = new HipoReader();
+            r.open(path.toString());
+            return r;
+        }
+        else {
+            evio = true;
+            EvioSource r = new EvioSource();
+            r.open(path.toString());
+            maxEvents = r.getEventCount();
+            decoder = new CLASDecoder4();
+            torus = opts.has("torus") ? opts.getDouble("torus") : null;
+            solenoid = opts.has("solenoid") ? opts.getDouble("solenoid") : null;
+            if (opts.has("variation")) decoder.setVariation(opts.getString("variation"));
+            if (opts.has("timestamp")) decoder.setTimestamp(opts.getString("timestamp"));
+            return r;
+        }
+    }
+
+    @Override
+    protected void closeReader() {
+        if (evio) ((EvioSource)reader).close();
+        else ((HipoReader)reader).close();
+    }
+
+    @Override
+    protected int readEventCount() throws EventReaderException {
+        if (evio) return ((EvioSource)reader).getEventCount();
+        else return ((HipoReader)reader).getEventCount();
+    }
+
+    @Override
+    protected Object readEvent(int eventNumber) throws EventReaderException {
+        try {
+            if (evio) {
+                if (eventNumber++ >= maxEvents) return null;
+                ByteBuffer b = ((EvioSource)reader).getEventBuffer(eventNumber, true);
+                EvioDataEvent e = new EvioDataEvent(b.array(), readByteOrder());
+                return decoder.getDecodedEvent(e, -1, eventNumber, torus, solenoid);
+            }
+            else {
+                return ((HipoReader)reader).getEvent(new Event(),eventNumber);
+            }
+        } catch (EvioException e) {
+            throw new EventReaderException(e);
+        }
+    }
+
+    @Override
+    public ByteOrder readByteOrder() throws EventReaderException {
+        return ByteOrder.LITTLE_ENDIAN;
+    }
+
+    @Override
+    protected EngineDataType getDataType() {
+        return Clas12Types.HIPO;
+    }
+
+}

common-tools/clara-io/src/main/java/org/jlab/io/clara/DecoderWriter.java

@@ -66,7 +66,7 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event
             init(opts);
             HipoWriterSorted w = new HipoWriterSorted();
             super.configure(w, opts);
-            w.open(file.toString());
+            w.open(file.toString().endsWith(".hipo") ? file.toString() : file.toString()+".hipo");
             return w;
         } catch (Exception e) {
             throw new EventWriterException(e);

common-tools/clara-io/src/main/resources/org/jlab/io/clara/Clas12Reader.yaml

@@ -0,0 +1,11 @@
+---
+name: Clas12Reader
+engine: org.jlab.io.clara.Clas12Reader
+type: java
+
+author: Nathan Baltzell
+email: baltzell@jlab.org
+
+version: 0.1
+description:
+  Reads EVIO or HIPO events from a file, converting to HIPO via "decoding" if EVIO.

common-tools/clas-detector/src/main/java/org/jlab/detector/base/DetectorDescriptor.java

@@ -88,6 +88,14 @@ public final void setSectorLayerComponent(int sector, int layer, int comp){
         this.dt_COMPONENT = comp;
     }
 
+    public final void setSectorLayerComponentOrderType(int sector, int layer, int comp, int order, int type) {
+        this.dt_SECTOR = sector;
+        this.dt_LAYER  = layer;
+        this.dt_COMPONENT = comp;
+        this.dt_ORDER = order;
+        this.detectorType = DetectorType.getType(type);
+    }
+
     public static int generateHashCode(int s, int l, int c){
         return  ((s<<24)&0xFF000000)|
                 ((l<<16)&0x00FF0000)|(c&0x0000FFFF);