# HG changeset patch
# User Raimund Renkert <rrenkert@intevation.de>
# Date 1377610101 -7200
# Node ID 5d11428e6a09267926a04ad31bcced86d045cd59
# Parent  046cdc094c7236ead01d8a8c1f68a62aeac3c9cc
Made the importer a little more robust and introduced a better warning/error reporting.

diff -r 046cdc094c72 -r 5d11428e6a09 src/main/java/de/intevation/lada/data/importer/AttributeMapper.java
--- a/src/main/java/de/intevation/lada/data/importer/AttributeMapper.java	Tue Aug 27 15:26:46 2013 +0200
+++ b/src/main/java/de/intevation/lada/data/importer/AttributeMapper.java	Tue Aug 27 15:28:21 2013 +0200
@@ -3,8 +3,12 @@
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -21,6 +25,7 @@
 import de.intevation.lada.model.LMesswert;
 import de.intevation.lada.model.LOrt;
 import de.intevation.lada.model.LProbe;
+import de.intevation.lada.model.LProbeInfo;
 import de.intevation.lada.model.SMessEinheit;
 import de.intevation.lada.model.SMessgroesse;
 import de.intevation.lada.rest.Response;
@@ -32,9 +37,20 @@
     private EntityManager em;
 
     @Inject
+    @Named("lproberepository")
+    private Repository probeRepo;
+
+    @Inject
     @Named("readonlyrepository")
     private Repository sRepository;
 
+    private List<ReportData> warnings;
+    private List<ReportData> errors;
+
+    public AttributeMapper() {
+        this.warnings = new ArrayList<ReportData>();
+        this.errors = new ArrayList<ReportData>();
+    }
     public LProbe addAttribute(String key, Object value, LProbe probe) {
         DateFormat format = new SimpleDateFormat("yyyyMMdd HHmm");
         if ("datenbasis_s".equals(key)) {
@@ -42,6 +58,10 @@
             probe.setDatenbasisId(v);
         }
         else if ("probe_id".equals(key)) {
+            if (probeRepo.findById(LProbeInfo.class, value.toString()) != null) {
+                errors.add(new ReportData("probe_id", value.toString(), 662));
+                return null;
+            }
             probe.setProbeId(value.toString());
         }
         else if ("hauptprobennummer".equals(key)) {
@@ -281,4 +301,23 @@
         }
         return ort;
     }
+
+    /**
+     * @return the warnings
+     */
+    public List<ReportData> getWarnings() {
+        return warnings;
+    }
+
+    /**
+     * @return the errors
+     */
+    public List<ReportData> getErrors() {
+        return errors;
+    }
+
+    public void reset() {
+        errors.clear();
+        warnings.clear();
+    }
 }
diff -r 046cdc094c72 -r 5d11428e6a09 src/main/java/de/intevation/lada/data/importer/Importer.java
--- a/src/main/java/de/intevation/lada/data/importer/Importer.java	Tue Aug 27 15:26:46 2013 +0200
+++ b/src/main/java/de/intevation/lada/data/importer/Importer.java	Tue Aug 27 15:28:21 2013 +0200
@@ -1,14 +1,14 @@
 package de.intevation.lada.data.importer;
 
+import java.util.List;
 import java.util.Map;
 
 import de.intevation.lada.auth.AuthenticationResponse;
-import de.intevation.lada.rest.Response;
 
 
 public interface Importer
 {
     public boolean importData(String content, AuthenticationResponse auth);
-    public Map<String, Map<String, Integer>> getErrors();
-    public Map<String, Map<String, Integer>> getWarnings();
+    public Map<String, List<ReportData>> getErrors();
+    public Map<String, List<ReportData>> getWarnings();
 }
diff -r 046cdc094c72 -r 5d11428e6a09 src/main/java/de/intevation/lada/data/importer/LAFImporter.java
--- a/src/main/java/de/intevation/lada/data/importer/LAFImporter.java	Tue Aug 27 15:26:46 2013 +0200
+++ b/src/main/java/de/intevation/lada/data/importer/LAFImporter.java	Tue Aug 27 15:28:21 2013 +0200
@@ -1,7 +1,9 @@
 package de.intevation.lada.data.importer;
 
 import java.math.BigInteger;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -69,30 +71,33 @@
     @Named("lmesswertrepository")
     private Repository messwertRepository;
 
-    private Map<String, Map<String, Integer>> warnings;
-    private Map<String, Map<String, Integer>> errors;
+    private Map<String, List<ReportData>> warnings;
+    private Map<String, List<ReportData>> errors;
 
     public LAFImporter() {
-        warnings = new HashMap<String, Map<String, Integer>>();
-        errors = new HashMap<String, Map<String, Integer>>();
+        warnings = new HashMap<String, List<ReportData>>();
+        errors = new HashMap<String, List<ReportData>>();
     }
 
     /**
      * @return the warnings
      */
-    public Map<String, Map<String, Integer>> getWarnings() {
+    public Map<String, List<ReportData>> getWarnings() {
         return warnings;
     }
 
     /**
      * @return the errors
      */
-    public Map<String, Map<String, Integer>> getErrors() {
+    public Map<String, List<ReportData>> getErrors() {
         return errors;
     }
 
     @Override
     public boolean importData(String content, AuthenticationResponse auth) {
+        this.warnings.clear();
+        this.errors.clear();
+        this.parser.reset();
         try {
             boolean success = parser.parse(content);
             if (success) {
@@ -110,20 +115,22 @@
                 writeMesswerte(auth, messwerte);
             }
             else {
-                Map<String, Integer> err = new HashMap<String, Integer>();
-                err.put("no success", 660);
-                errors.put("parser", err);
+                List<ReportData> report = new ArrayList<ReportData>();
+                report.add( new ReportData("parser", "no success", 660));
+                errors.put("parser", report);
                 return false;
             }
         }
         catch (LAFParserException e) {
-            Map<String, Integer> err = new HashMap<String, Integer>();
-            err.put(e.getMessage(), 660);
-            errors.put("parser", err);
+            List<ReportData> report = new ArrayList<ReportData>();
+            report.add(new ReportData("parser", e.getMessage(), 660));
+            errors.put("parser", report);
             return false;
         }
-        Map<String, Map<String, Map<String, Integer>>> data =
-            new HashMap<String, Map<String,Map<String, Integer>>>();
+        Map<String, Map<String, List<ReportData>>> data =
+            new HashMap<String, Map<String, List<ReportData>>>();
+        this.warnings.putAll(this.parser.getWarnings());
+        this.errors.putAll(this.parser.getErrors());
         data.put("warnings", warnings);
         data.put("errors", errors);
         return true;
@@ -140,16 +147,17 @@
                     messungValidator.validate(messung, false);
                 messungRepository.create(messung);
                 if (warn != null) {
-                    warnings.put(
-                        messung.getMessungsId().toString(),
-                        warn);
+                    for (String key : warn.keySet()) {
+                       // warnings.put(messung.getProbeId(),
+                       // new ReportData(key, "", warn.get(key)));
+                    }
                 }
             }
             catch (ValidationException e) {
-                errors.put(messung.getProbeId(), e.getErrors());
-                warnings.put(
-                    messung.getProbeId(),
-                    e.getWarnings());
+                //errors.put(messung.getProbeId(), e.getErrors());
+                //warnings.put(
+                //    messung.getProbeId(),
+                //    e.getWarnings());
             }
         }
     }
@@ -165,16 +173,16 @@
                     messwertValidator.validate(messwert, false);
                 Response r = messwertRepository.create(messwert);
                 if (warn != null) {
-                    warnings.put(
-                        messwert.getProbeId(),
-                        warn);
+                //    warnings.put(
+                //        messwert.getProbeId(),
+                //        warn);
                 }
             }
             catch (ValidationException e) {
-                errors.put(messwert.getProbeId(), e.getErrors());
-                warnings.put(
-                    messwert.getProbeId(),
-                    e.getWarnings());
+                //errors.put(messwert.getProbeId(), e.getErrors());
+                //warnings.put(
+                //    messwert.getProbeId(),
+                //    e.getWarnings());
             }
         }
     }
@@ -193,7 +201,7 @@
                 err.put(
                     kommentar.getProbeId() + " - " +
                     kommentar.getkId(), 661);
-                errors.put("lkommentarp", err);
+                //errors.put("lkommentarp", err);
             }
         }
     }
@@ -224,14 +232,14 @@
                     ortValidator.validate(ort, false);
                 ortRepository.create(ort);
                 if (warn != null) {
-                    warnings.put(String.valueOf(ort.getOrtId()), warn);
+                  //  warnings.put(String.valueOf(ort.getOrtId()), warn);
                 }
             }
             catch (ValidationException e) {
-                errors.put(String.valueOf(ort.getOrtId()), e.getErrors());
-                warnings.put(
-                    String.valueOf(ort.getOrtId()),
-                    e.getWarnings());
+                //errors.put(String.valueOf(ort.getOrtId()), e.getErrors());
+                //warnings.put(
+                //    String.valueOf(ort.getOrtId()),
+                //    e.getWarnings());
             }
         }
     }
@@ -242,19 +250,19 @@
             if (!authorized(probe, auth)) {
                 Map<String, Integer> err = new HashMap<String, Integer>();
                 err.put("not authorized", 699);
-                errors.put(probe.getProbeId(), err);
+                //errors.put(probe.getProbeId(), err);
                 continue;
             }
             try {
                 Map<String, Integer> warn =
                     probeValidator.validate(probe, false);
                 if (warn != null) {
-                    warnings.put(probe.getProbeId(), warn);
+                  //  warnings.put(probe.getProbeId(), warn);
                 }
             }
             catch (ValidationException e) {
-                errors.put(probe.getProbeId(), e.getErrors());
-                warnings.put(probe.getProbeId(), e.getWarnings());
+                //errors.put(probe.getProbeId(), e.getErrors());
+                //warnings.put(probe.getProbeId(), e.getWarnings());
                 continue;
             }
             persist(probe);
diff -r 046cdc094c72 -r 5d11428e6a09 src/main/java/de/intevation/lada/data/importer/LAFParser.java
--- a/src/main/java/de/intevation/lada/data/importer/LAFParser.java	Tue Aug 27 15:26:46 2013 +0200
+++ b/src/main/java/de/intevation/lada/data/importer/LAFParser.java	Tue Aug 27 15:28:21 2013 +0200
@@ -1,7 +1,10 @@
 package de.intevation.lada.data.importer;
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -32,7 +35,12 @@
     List<LKommentarP> probeKommentare;
     List<LKommentarM> messungKommentare;
 
+    private Map<String, List<ReportData>> warnings;
+    private Map<String, List<ReportData>> errors;
+
     public LAFParser() {
+        this.warnings = new HashMap<String, List<ReportData>>();
+        this.errors = new HashMap<String, List<ReportData>>();
         this.setDryRun(false);
         //this.producer = new LAFProducer();
         this.proben = new ArrayList<LProbe>();
@@ -64,11 +72,25 @@
             if (nextPos > 0) {
                 single = laf.substring(0, nextPos + 1);
                 laf = laf.substring(nextPos + 1);
-                readAll(single);
+                try {
+                    readAll(single);
+                }
+                catch (LAFParserException lpe) {
+                    this.errors.putAll(producer.getErrors());
+                    this.producer.reset();
+                    continue;
+                }
             }
             else {
-                readAll(laf);
-                laf = "";
+                try {
+                    readAll(laf);
+                    laf = "";
+                }
+                catch (LAFParserException lpe) {
+                    this.errors.putAll(producer.getErrors());
+                    laf = "";
+                    continue;
+                }
             }
             if (!this.dryRun) {
                 proben.add(producer.getProbe());
@@ -235,4 +257,24 @@
     public List<LKommentarM> getMessungKommentare() {
         return messungKommentare;
     }
+
+    /**
+     * @return the warnings
+     */
+    public Map<String, List<ReportData>> getWarnings() {
+        return warnings;
+    }
+
+    /**
+     * @return the errors
+     */
+    public Map<String, List<ReportData>> getErrors() {
+        return errors;
+    }
+
+    public void reset() {
+        producer.reset();
+        this.errors.clear();
+        this.warnings.clear();
+    }
 }
diff -r 046cdc094c72 -r 5d11428e6a09 src/main/java/de/intevation/lada/data/importer/LAFProducer.java
--- a/src/main/java/de/intevation/lada/data/importer/LAFProducer.java	Tue Aug 27 15:26:46 2013 +0200
+++ b/src/main/java/de/intevation/lada/data/importer/LAFProducer.java	Tue Aug 27 15:28:21 2013 +0200
@@ -1,7 +1,10 @@
 package de.intevation.lada.data.importer;
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -36,10 +39,15 @@
     private List<EntryFormat> messungFormat;
     private List<EntryFormat> ortFormat;
 
+    private Map<String, List<ReportData>> warnings;
+    private Map<String, List<ReportData>> errors;
+
     @Inject
     private AttributeMapper mapper;
 
     public LAFProducer() {
+        this.warnings = new HashMap<String, List<ReportData>>();
+        this.errors = new HashMap<String, List<ReportData>>();
         this.probe = new LProbe();
         this.pKommentare = new ArrayList<LKommentarP>();
         this.mKommentare = new ArrayList<LKommentarM>();
@@ -55,7 +63,8 @@
     }
 
     @Override
-    public void addData(String key, Object values) {
+    public void addData(String key, Object values)
+    throws LAFParserException {
         String lKey = key.toLowerCase();
         if(lKey.equals("probenkommentar")) {
             LKommentarP kommentar = new LKommentarP();
@@ -98,6 +107,10 @@
         }
         else if (isValidProbe(lKey, values.toString())) {
             this.probe = mapper.addAttribute(lKey, values, this.probe);
+            if (this.probe == null) {
+                this.errors.put(values.toString(), mapper.getErrors());
+                throw new LAFParserException(values.toString() + " exists");
+            }
         }
         else if (isValidOrt(lKey, values.toString())) {
             this.ort = mapper.addAttribute(lKey, values, this.ort);
@@ -169,6 +182,8 @@
 
     @Override
     public void reset() {
+        this.errors.clear();
+        this.warnings.clear();
         this.probe = new LProbe();
         this.messungen.clear();
         this.messung = null;
@@ -177,6 +192,7 @@
         this.messwerte.clear();
         this.mKommentare.clear();
         this.pKommentare.clear();
+        mapper.reset();
     }
 
     public void newMessung() {
@@ -197,4 +213,18 @@
         this.ort = new LOrt();
         this.ort.setProbeId(this.probe.getProbeId());
     }
+
+    /**
+     * @return the warnings
+     */
+    public Map<String, List<ReportData>> getWarnings() {
+        return this.warnings;
+    }
+
+    /**
+     * @return the errors
+     */
+    public Map<String, List<ReportData>> getErrors() {
+        return this.errors;
+    }
 }
diff -r 046cdc094c72 -r 5d11428e6a09 src/main/java/de/intevation/lada/data/importer/Producer.java
--- a/src/main/java/de/intevation/lada/data/importer/Producer.java	Tue Aug 27 15:26:46 2013 +0200
+++ b/src/main/java/de/intevation/lada/data/importer/Producer.java	Tue Aug 27 15:28:21 2013 +0200
@@ -1,6 +1,7 @@
 package de.intevation.lada.data.importer;
 
 import java.util.List;
+import java.util.Map;
 
 import de.intevation.lada.model.LKommentarM;
 import de.intevation.lada.model.LKommentarP;
@@ -12,7 +13,8 @@
 
 public interface Producer
 {
-    public void addData(String key, Object values);
+    public void addData(String key, Object values)
+        throws LAFParserException;
     public LProbe getProbe();
     public List<LMessung> getMessungen();
     public List<LOrt> getOrte();
@@ -22,4 +24,6 @@
     public void reset();
     public void newMessung();
     public void newOrt();
+    public Map<String, List<ReportData>> getErrors();
+    public Map<String, List<ReportData>> getWarnings();
 }
diff -r 046cdc094c72 -r 5d11428e6a09 src/main/java/de/intevation/lada/rest/LAFImportService.java
--- a/src/main/java/de/intevation/lada/rest/LAFImportService.java	Tue Aug 27 15:26:46 2013 +0200
+++ b/src/main/java/de/intevation/lada/rest/LAFImportService.java	Tue Aug 27 15:28:21 2013 +0200
@@ -88,12 +88,10 @@
             }
 
             boolean success = importer.importData(content, auth);
-            List<Object> respData = new LinkedList<Object>();
-            respData.add(importer.getErrors());
-            respData.add(importer.getWarnings());
-            Map<String, String> fileData = new HashMap<String, String>();
-            fileData.put("filename", name);
-            respData.add(fileData);
+            Map<String, Object> respData = new HashMap<String,Object>();
+            respData.put("errors", importer.getErrors());
+            respData.put("warnings", importer.getWarnings());
+            respData.put("filename", name);
             int code = 200;
             if (!success) {
                 code = 660;