# HG changeset patch
# User Raimund Renkert <rrenkert@intevation.de>
# Date 1377007997 -7200
# Node ID 821557a17e5ed3b90d3638bc7551c72f65d8630b
# Parent  ae4bf396bd3b8d909953e78f19a9cd2c60091844
First version of the LAF importer.

The importer module currently only runs with the test application (comming in the next commit)!
* LAF Parser:
  - Uses a small implementation of a state machine.
  - Extracts the keys with its value or multi value.
  - Uses the producer interface to generate objects.
* Attribute mapper:
  - Maps the attributes defined in the configuration file to object attributes.
  - Generates objects from multi value attributes.
* LAF format:
  - Reads the config file
* LAF importer:
  - Implemetation of the importer interface for LAF format.

diff -r ae4bf396bd3b -r 821557a17e5e src/main/java/de/intevation/lada/data/importer/AttributeMapper.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/intevation/lada/data/importer/AttributeMapper.java	Tue Aug 20 16:13:17 2013 +0200
@@ -0,0 +1,238 @@
+package de.intevation.lada.data.importer;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.inject.Inject;
+import javax.persistence.EntityManager;
+
+import de.intevation.lada.model.LKommentarM;
+import de.intevation.lada.model.LKommentarP;
+import de.intevation.lada.model.LMessung;
+import de.intevation.lada.model.LMessungId;
+import de.intevation.lada.model.LMesswert;
+import de.intevation.lada.model.LOrt;
+import de.intevation.lada.model.LProbe;
+
+
+public class AttributeMapper
+{
+    @Inject
+    private EntityManager em;
+
+    public LProbe addAttribute(String key, Object value, LProbe probe) {
+        DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm");
+        if ("datenbasis_s".equals(key)) {
+            Integer v = Integer.valueOf(value.toString());
+            probe.setDatenbasisId(v);
+        }
+        else if ("probe_id".equals(key)) {
+            probe.setProbeId(value.toString());
+        }
+        else if ("hauptprobennummer".equals(key)) {
+            probe.setHauptprobenNr(value.toString());
+        }
+        else if ("mpr_id".equals(key)) {
+            Integer v = Integer.valueOf(value.toString());
+            probe.setMprId(v);
+        }
+        else if ("messprogramm_land".equals(key)) {
+            probe.setMplId(value.toString());
+        }
+        else if ("messstelle".equals(key)) {
+            probe.setMstId(value.toString());
+        }
+        else if ("betriebsart".equals(key)) {
+            probe.setBaId(value.toString());
+        }
+        else if ("soll_datum_uhrzeit_a".equals(key)) {
+            try {
+                Date d = format.parse(value.toString());
+                probe.setSolldatumBeginn(d);
+            }
+            catch (ParseException e) {
+                //TODO handle warning.
+            }
+        }
+        else if ("soll_datum_uhrzeit_e".equals(key)) {
+            try {
+                Date d = format.parse(value.toString());
+                probe.setSolldatumEnde(d);
+            }
+            catch (ParseException e) {
+                //TODO handle warning.
+            }
+        }
+        else if ("probenahme_datum_uhrzeit_a".equals(key)) {
+            try {
+                Date d = format.parse(value.toString());
+                probe.setProbeentnahmeBeginn(d);
+            }
+            catch (ParseException e) {
+                //TODO handle warning.
+            }
+        }
+        else if ("probenahme_datum_uhrzeit_e".equals(key)) {
+            try {
+                Date d = format.parse(value.toString());
+                probe.setProbeentnahmeEnde(d);
+            }
+            catch (ParseException e) {
+                //TODO handle warning.
+            }
+        }
+        else if ("umweltbereich_s".equals(key)) {
+            probe.setUmwId(value.toString());
+        }
+        else if ("deskriptoren".equals(key)) {
+            probe.setMediaDesk(value.toString());
+        }
+        else if ("testdaten".equals(key)) {
+            if (!value.toString().equals("0")) {
+                probe.setTest(true);
+            }
+            else {
+                probe.setTest(false);
+            }
+        }
+        return probe;
+    }
+
+    public LKommentarP addAttribute(
+        String key,
+        Object values,
+        LKommentarP kommentar
+    ) {
+        DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm");
+        String v = values.toString();
+        String erzeuger = v.substring(1, 5);
+        String date = v.substring(8, 21);
+        Date d;
+        try {
+            d = format.parse(date);
+            kommentar.setKDatum(d);
+        }
+        catch (ParseException e) {
+            //TODO: handle warning.
+        }
+        String text = v.substring(23, v.length() -1);
+        kommentar.setErzeuger(erzeuger);
+        kommentar.setKText(text);
+        return kommentar;
+    }
+
+    public LKommentarM addAttribute(
+        String key,
+        Object values,
+        LKommentarM kommentar
+    ) {
+        DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm");
+        String v = values.toString();
+        String erzeuger = v.substring(1, 5);
+        String date = v.substring(8, 21);
+        Date d;
+        try {
+            d = format.parse(date);
+            kommentar.setKDatum(d);
+        }
+        catch (ParseException e) {
+            //TODO: handle warning.
+        }
+        String text = v.substring(23, v.length() -1);
+        kommentar.setErzeuger(erzeuger);
+        kommentar.setKText(text);
+        return kommentar;
+    }
+
+    public LMessung addAttribute(
+        String key,
+        Object values,
+        LMessung messung
+    ) {
+        DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm");
+        if ("messungs_id".equals(key)) {
+            LMessungId id = messung.getId();
+            Integer v = Integer.valueOf(values.toString());
+            id.setMessungsId(v);
+            messung.setId(id);
+        }
+        else if ("nebenprobennummer".equals(key)) {
+            messung.setNebenprobenNr(values.toString());
+        }
+        else if ("mess_datum_uhrzeit".equals(key)) {
+            try {
+                Date d = format.parse(values.toString());
+                messung.setMesszeitpunkt(d);
+            }
+            catch (ParseException e) {
+                //TODO: handle warnings.
+            }
+        }
+        else if ("messzeit_sekunden".equals(key)) {
+            Integer i = Integer.valueOf(values.toString());
+            messung.setMessdauer(i);
+        }
+        else if ("messmethode_s".equals(key)) {
+            messung.setMmtId(values.toString());
+        }
+        else if ("bearbeitungsstatus".equals(key)) {
+            //ignored.!?
+        }
+        else if ("erfassung_abgeschlossen".equals(key)) {
+            if(!values.toString().equals("0")) {
+                messung.setFertig(true);
+            }
+            else {
+                messung.setFertig(false);
+            }
+        }
+        return messung;
+    }
+
+    public LMesswert addAttribute(
+        String key,
+        Object values,
+        LMesswert messwert
+    ) {
+        Pattern p = Pattern.compile(
+            "(\".+\")( .+ )(\".+\")( .*)( .{1,12})( .{1,9})(.{0,9})(.{0,3})");
+        //TODO Does not perfectly match... Use better matching for floats.
+        Matcher m = p.matcher(values.toString());
+        if (m.matches()) {
+            String messgroesse = m.group(1);
+            String wert = m.group(2);
+            String einheit = m.group(3);
+            if (wert.startsWith(" <")) {
+                wert = wert.substring(2);
+                messwert.setGrenzwertueberschreitung(false);
+            }
+            else if (wert.startsWith(" >")) {
+                wert = wert.substring(2);
+                messwert.setGrenzwertueberschreitung(true);
+            }
+            float fWert = Float.valueOf(wert);
+            messwert.setMesswert(fWert);
+        }
+        //TODO: Match the other values.
+        return messwert;
+    }
+
+    public LOrt addAttribute(
+        String key,
+        Object values,
+        LOrt ort
+    ) {
+        if ("ort_id".equals(key)) {
+            Integer v = Integer.valueOf(values.toString());
+            ort.setOrtId(v);
+        }
+        else if ("ort_typ".equals(key)) {
+            ort.setOrtsTyp(values.toString());
+        }
+        return ort;
+    }
+}
diff -r ae4bf396bd3b -r 821557a17e5e src/main/java/de/intevation/lada/data/importer/EntryFormat.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/intevation/lada/data/importer/EntryFormat.java	Tue Aug 20 16:13:17 2013 +0200
@@ -0,0 +1,38 @@
+package de.intevation.lada.data.importer;
+
+import java.util.regex.Pattern;
+
+
+public class EntryFormat
+{
+    private String key;
+    private Pattern pattern;
+    private Object defaultValue;
+
+    public EntryFormat() {
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public void setKey(String key) {
+        this.key = key;
+    }
+
+    public Pattern getPattern() {
+        return pattern;
+    }
+
+    public void setPattern(Pattern pattern) {
+        this.pattern = pattern;
+    }
+
+    public Object getDefaultValue() {
+        return defaultValue;
+    }
+
+    public void setDefaultValue(Object defaultValue) {
+        this.defaultValue = defaultValue;
+    }
+}
diff -r ae4bf396bd3b -r 821557a17e5e src/main/java/de/intevation/lada/data/importer/LAFFormat.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/intevation/lada/data/importer/LAFFormat.java	Tue Aug 20 16:13:17 2013 +0200
@@ -0,0 +1,62 @@
+package de.intevation.lada.data.importer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import javax.ejb.Singleton;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+
+public class LAFFormat
+{
+    private JSONObject fileContent;
+
+    public boolean readConfigFile(String fileName) {
+        try {
+            byte[] encoded = Files.readAllBytes(Paths.get(fileName));
+            Charset encoding = Charset.defaultCharset();
+            String content =
+                encoding.decode(ByteBuffer.wrap(encoded)).toString();
+            fileContent = new JSONObject(content);
+            return true;
+        }
+        catch (IOException ioe) {
+            return false;
+        }
+        catch (JSONException je) {
+            return false;
+        }
+    }
+
+    public List<EntryFormat> getFormat(String dataType) {
+        List<EntryFormat> formats = new LinkedList<EntryFormat>();
+        try {
+            JSONArray block = fileContent.getJSONArray(dataType);
+            for (int i = 0; i < block.length(); i++) {
+                JSONObject jEntry = block.getJSONObject(i);
+                EntryFormat entry = new EntryFormat();
+                entry.setKey(jEntry.getString("key"));
+                Pattern pattern =
+                    Pattern.compile(
+                        jEntry.getString("regex"),
+                        Pattern.MULTILINE);
+                entry.setPattern(pattern);
+                entry.setDefaultValue(jEntry.get("default"));
+                formats.add(entry);
+            }
+            return formats;
+        }
+        catch (JSONException e) {
+            return null;
+        }
+    }
+}
diff -r ae4bf396bd3b -r 821557a17e5e src/main/java/de/intevation/lada/data/importer/LAFImporter.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/intevation/lada/data/importer/LAFImporter.java	Tue Aug 20 16:13:17 2013 +0200
@@ -0,0 +1,16 @@
+package de.intevation.lada.data.importer;
+
+import javax.inject.Named;
+
+import de.intevation.lada.auth.AuthenticationResponse;
+import de.intevation.lada.rest.Response;
+
+@Named("lafimporter")
+public class LAFImporter
+implements Importer
+{
+    @Override
+    public Response importData(String content, AuthenticationResponse auth) {
+        return new Response(true, 200, null);
+    }
+}
diff -r ae4bf396bd3b -r 821557a17e5e src/main/java/de/intevation/lada/data/importer/LAFParser.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/intevation/lada/data/importer/LAFParser.java	Tue Aug 20 16:13:17 2013 +0200
@@ -0,0 +1,181 @@
+package de.intevation.lada.data.importer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import de.intevation.lada.model.LKommentarM;
+import de.intevation.lada.model.LKommentarP;
+import de.intevation.lada.model.LMessung;
+import de.intevation.lada.model.LMesswert;
+import de.intevation.lada.model.LOrt;
+import de.intevation.lada.model.LProbe;
+
+
+public class LAFParser {
+
+    private static final String PROBE_NEXT = "\n%PROBE%";
+
+    private boolean dryRun;
+
+    //@Inject
+    //@Named("lafproducer")
+    private Producer producer;
+
+    List<LProbe> proben;
+    List<LMessung> messungen;
+    List<LOrt> orte;
+    List<LMesswert> messwerte;
+    List<LKommentarP> probeKommentare;
+    List<LKommentarM> messungKommentare;
+
+    public LAFParser() {
+        this.setDryRun(false);
+        this.producer = new LAFProducer();
+        this.proben = new ArrayList<LProbe>();
+        this.messungen = new ArrayList<LMessung>();
+        this.orte = new ArrayList<LOrt>();
+        this.messwerte = new ArrayList<LMesswert>();
+        this.probeKommentare = new ArrayList<LKommentarP>();
+        this.messungKommentare = new ArrayList<LKommentarM>();
+    }
+
+    public boolean parse(String laf)
+    throws LAFParserException
+    {
+        boolean parsed = false;
+        while (laf.startsWith("%PROBE%\n")) {
+            parsed = true;
+            int nextPos = laf.indexOf(PROBE_NEXT);
+            String single = "";
+            if (nextPos > 0) {
+                single = laf.substring(0, nextPos + 1);
+                laf = laf.substring(nextPos + 1);
+                readAll(single);
+            }
+            else {
+                readAll(laf);
+                laf = "";
+            }
+            if (!this.dryRun) {
+                proben.add(producer.getProbe());
+                messungen.addAll(producer.getMessungen());
+                orte.addAll(producer.getOrte());
+                messwerte.addAll(producer.getMesswerte());
+                probeKommentare.addAll(producer.getProbenKommentare());
+                messungKommentare.addAll(producer.getMessungsKommentare());
+                producer.reset();
+            }
+        }
+        if (!parsed) {
+            throw new LAFParserException("No %PROBE% at the begining.");
+        }
+        return parsed;
+    }
+
+    private void readAll(String content)
+    throws LAFParserException
+    {
+        boolean key = false;
+        boolean value = false;
+        boolean header = false;
+        boolean white = false;
+        boolean string = false;
+        boolean multiValue = false;
+        String keyString = "";
+        String valueString = "";
+        String headerString = "";
+        for (int i = 0; i < content.length(); i++) {
+            char current = content.charAt(i);
+
+            if ((current == '"' || (current == ' ' && !string)) &&
+                value &&
+                i < content.length() - 1 &&
+                (content.charAt(i + 1) != '\n' &&
+                content.charAt(i + 1) != '\r')) {
+                multiValue = true;
+            }
+
+            if (current == '"' && !string) {
+                string = true;
+            }
+            else if (current == '"' && string) {
+                string = false;
+            }
+
+            if (current == ' ' && !value) {
+                key = false;
+                white = true;
+                continue;
+            }
+            else if (current != ' ' && white) {
+                value = true;
+                white = false;
+            }
+            else if (current == '%' && !header && !value) {
+                headerString = "";
+
+                key = false;
+                header = true;
+            }
+            else if ((current == '\n' || current == '\r') && header) {
+                header = false;
+                key = true;
+                if (!dryRun) {
+                    if (headerString.contains("MESSUNG")) {
+                        producer.newMessung();
+                    }
+                    if (headerString.contains("ORT")) {
+                        producer.newOrt();
+                    }
+                }
+                continue;
+            }
+            else if (current == '"' && !value) {
+                value = true;
+            }
+            else if ((current == '\n' || current == '\r') && value && !string) {
+                if (!multiValue && valueString.startsWith("\"")) {
+                    valueString =
+                        valueString.substring(1, valueString.length() - 1);
+                }
+                value = false;
+                multiValue = false;
+                key = true;
+                if (!this.dryRun) {
+                    producer.addData(keyString, valueString);
+                }
+                keyString = "";
+                valueString = "";
+                continue;
+            }
+            else if ((current == '\n' || current == '\r') && key) {
+                throw new LAFParserException("No value for key: " + keyString);
+            }
+
+            if (key) {
+                keyString += current;
+            }
+            else if (value) {
+                valueString += current;
+            }
+            else if (header) {
+                headerString += current;
+            }
+        }
+        if (!dryRun) {
+            this.producer.newMessung();
+            this.producer.newOrt();
+        }
+    }
+
+    public boolean isDryRun() {
+        return dryRun;
+    }
+
+    public void setDryRun(boolean dryRun) {
+        this.dryRun = dryRun;
+    }
+}
diff -r ae4bf396bd3b -r 821557a17e5e src/main/java/de/intevation/lada/data/importer/LAFParserException.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/intevation/lada/data/importer/LAFParserException.java	Tue Aug 20 16:13:17 2013 +0200
@@ -0,0 +1,11 @@
+package de.intevation.lada.data.importer;
+
+
+public class LAFParserException
+extends Exception
+{
+    
+    public LAFParserException(String message) {
+        super(message);
+    }
+}
diff -r ae4bf396bd3b -r 821557a17e5e src/main/java/de/intevation/lada/data/importer/LAFProducer.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/intevation/lada/data/importer/LAFProducer.java	Tue Aug 20 16:13:17 2013 +0200
@@ -0,0 +1,184 @@
+package de.intevation.lada.data.importer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import javax.inject.Named;
+
+import de.intevation.lada.model.LKommentarM;
+import de.intevation.lada.model.LKommentarP;
+import de.intevation.lada.model.LMessung;
+import de.intevation.lada.model.LMessungId;
+import de.intevation.lada.model.LMesswert;
+import de.intevation.lada.model.LMesswertId;
+import de.intevation.lada.model.LOrt;
+import de.intevation.lada.model.LProbe;
+
+@Named("lafproducer")
+public class LAFProducer
+implements Producer
+{
+
+    private LProbe probe;
+    private LMessung messung;
+    private LOrt ort;
+
+    private List<LKommentarP> pKommentare;
+    private List<LKommentarM> mKommentare;
+    private List<LMessung> messungen;
+    private List<LOrt> orte;
+    private List<LMesswert> messwerte;
+
+    private List<EntryFormat> probenFormat;
+    private List<EntryFormat> messungFormat;
+    private List<EntryFormat> ortFormat;
+
+    private AttributeMapper mapper;
+
+    public LAFProducer() {
+        this.mapper = new AttributeMapper();
+        this.probe = new LProbe();
+        this.pKommentare = new ArrayList<LKommentarP>();
+        this.mKommentare = new ArrayList<LKommentarM>();
+        this.messungen = new ArrayList<LMessung>();
+        this.orte = new ArrayList<LOrt>();
+        this.messwerte = new ArrayList<LMesswert>();
+        String fileName = System.getProperty("de_intevation_lada_import");
+        LAFFormat format = new LAFFormat();
+        format.readConfigFile(fileName);
+        probenFormat = format.getFormat("probe");
+        messungFormat = format.getFormat("messung");
+        ortFormat = format.getFormat("ort");
+    }
+
+    @Override
+    public void addData(String key, Object values) {
+        String lKey = key.toLowerCase();
+        if(lKey.equals("probenkommentar")) {
+            this.pKommentare.add(
+                mapper.addAttribute(lKey, values, new LKommentarP()));
+        }
+        else if (lKey.equals("kommentar")) {
+            this.mKommentare.add(
+                mapper.addAttribute(lKey, values, new LKommentarM()));
+        }
+        else if (lKey.equals("probenzusatzbeschreibung")) {
+            //TODO: implement this!
+        }
+        else if (lKey.equals("pzb_s")) {
+            //TODO: implement this!
+        }
+        else if (lKey.equals("messwert")) {
+            LMesswertId id = new LMesswertId();
+            id.setProbeId(this.probe.getProbeId());
+            id.setMessungsId(this.messung.getId().getMessungsId());
+            LMesswert m = new LMesswert();
+            m.setId(id);
+            this.messwerte.add(
+                mapper.addAttribute(lKey, values, m));
+        }
+        else if (isValidMessung(lKey, values.toString())) {
+            this.messung = mapper.addAttribute(lKey, values, this.messung);
+        }
+        else if (isValidProbe(lKey, values.toString())) {
+            this.probe = mapper.addAttribute(lKey, values, this.probe);
+        }
+        else if (isValidOrt(lKey, values.toString())) {
+            this.ort = mapper.addAttribute(lKey, values, this.ort);
+        }
+    }
+
+    private boolean isValidOrt(String key, String value) {
+        for (EntryFormat ef: ortFormat) {
+            if (ef.getKey().equals(key.toLowerCase())) {
+                if (ef.getPattern().matcher(value).matches()) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    private boolean isValidMessung(String key, String value) {
+        for (EntryFormat ef: messungFormat) {
+            if (ef.getKey().equals(key.toLowerCase())) {
+                if (ef.getPattern().matcher(value).matches()) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    private boolean isValidProbe(String key, String value) {
+        for (EntryFormat ef: probenFormat) {
+            if (ef.getKey().equals(key.toLowerCase())) {
+                if (ef.getPattern().matcher(value).matches()) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public LProbe getProbe() {
+        return this.probe;
+    }
+
+    @Override
+    public List<LMessung> getMessungen() {
+        return this.messungen;
+    }
+
+    @Override
+    public List<LOrt> getOrte() {
+        return this.orte;
+    }
+
+    @Override
+    public List<LKommentarP> getProbenKommentare() {
+        return this.pKommentare;
+    }
+
+    @Override
+    public List<LKommentarM> getMessungsKommentare() {
+        return this.mKommentare;
+    }
+
+    @Override
+    public List<LMesswert> getMesswerte() {
+        return this.messwerte;
+    }
+
+    @Override
+    public void reset() {
+        this.probe = new LProbe();
+        this.messungen.clear();
+        this.messung = null;
+        this.orte.clear();
+        this.ort = null;
+        this.messwerte.clear();
+        this.mKommentare.clear();
+        this.pKommentare.clear();
+    }
+
+    public void newMessung() {
+        if (this.messung != null) {
+            this.messungen.add(this.messung);
+        }
+        LMessungId id = new LMessungId();
+        id.setProbeId(this.probe.getProbeId());
+        this.messung = new LMessung();
+        this.messung.setId(id);
+    }
+
+    public void newOrt() {
+        if (this.ort != null) {
+            this.orte.add(this.ort);
+        }
+        this.ort = new LOrt();
+        this.ort.setProbeId(this.probe.getProbeId());
+    }
+}