Mercurial > lada > lada-server
changeset 331:5d11428e6a09
Made the importer a little more robust and introduced a better warning/error reporting.
| author | Raimund Renkert <rrenkert@intevation.de> |
|---|---|
| date | Tue, 27 Aug 2013 15:28:21 +0200 |
| parents | 046cdc094c72 |
| children | 22fc8b2939eb |
| files | src/main/java/de/intevation/lada/data/importer/AttributeMapper.java src/main/java/de/intevation/lada/data/importer/Importer.java src/main/java/de/intevation/lada/data/importer/LAFImporter.java src/main/java/de/intevation/lada/data/importer/LAFParser.java src/main/java/de/intevation/lada/data/importer/LAFProducer.java src/main/java/de/intevation/lada/data/importer/Producer.java src/main/java/de/intevation/lada/rest/LAFImportService.java |
| diffstat | 7 files changed, 173 insertions(+), 52 deletions(-) [+] |
line wrap: on
line diff
--- a/src/main/java/de/intevation/lada/data/importer/AttributeMapper.java Tue Aug 27 15:26:46 2013 +0200 +++ b/src/main/java/de/intevation/lada/data/importer/AttributeMapper.java Tue Aug 27 15:28:21 2013 +0200 @@ -3,8 +3,12 @@ import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,6 +25,7 @@ import de.intevation.lada.model.LMesswert; import de.intevation.lada.model.LOrt; import de.intevation.lada.model.LProbe; +import de.intevation.lada.model.LProbeInfo; import de.intevation.lada.model.SMessEinheit; import de.intevation.lada.model.SMessgroesse; import de.intevation.lada.rest.Response; @@ -32,9 +37,20 @@ private EntityManager em; @Inject + @Named("lproberepository") + private Repository probeRepo; + + @Inject @Named("readonlyrepository") private Repository sRepository; + private List<ReportData> warnings; + private List<ReportData> errors; + + public AttributeMapper() { + this.warnings = new ArrayList<ReportData>(); + this.errors = new ArrayList<ReportData>(); + } public LProbe addAttribute(String key, Object value, LProbe probe) { DateFormat format = new SimpleDateFormat("yyyyMMdd HHmm"); if ("datenbasis_s".equals(key)) { @@ -42,6 +58,10 @@ probe.setDatenbasisId(v); } else if ("probe_id".equals(key)) { + if (probeRepo.findById(LProbeInfo.class, value.toString()) != null) { + errors.add(new ReportData("probe_id", value.toString(), 662)); + return null; + } probe.setProbeId(value.toString()); } else if ("hauptprobennummer".equals(key)) { @@ -281,4 +301,23 @@ } return ort; } + + /** + * @return the warnings + */ + public List<ReportData> getWarnings() { + return warnings; + } + + /** + * @return the errors + */ + public List<ReportData> getErrors() { + return errors; + } + + public void reset() { + errors.clear(); + warnings.clear(); + } }
--- a/src/main/java/de/intevation/lada/data/importer/Importer.java Tue Aug 27 15:26:46 2013 +0200 +++ b/src/main/java/de/intevation/lada/data/importer/Importer.java Tue Aug 27 15:28:21 2013 +0200 @@ -1,14 +1,14 @@ package de.intevation.lada.data.importer; +import java.util.List; import java.util.Map; import de.intevation.lada.auth.AuthenticationResponse; -import de.intevation.lada.rest.Response; public interface Importer { public boolean importData(String content, AuthenticationResponse auth); - public Map<String, Map<String, Integer>> getErrors(); - public Map<String, Map<String, Integer>> getWarnings(); + public Map<String, List<ReportData>> getErrors(); + public Map<String, List<ReportData>> getWarnings(); }
--- a/src/main/java/de/intevation/lada/data/importer/LAFImporter.java Tue Aug 27 15:26:46 2013 +0200 +++ b/src/main/java/de/intevation/lada/data/importer/LAFImporter.java Tue Aug 27 15:28:21 2013 +0200 @@ -1,7 +1,9 @@ package de.intevation.lada.data.importer; import java.math.BigInteger; +import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -69,30 +71,33 @@ @Named("lmesswertrepository") private Repository messwertRepository; - private Map<String, Map<String, Integer>> warnings; - private Map<String, Map<String, Integer>> errors; + private Map<String, List<ReportData>> warnings; + private Map<String, List<ReportData>> errors; public LAFImporter() { - warnings = new HashMap<String, Map<String, Integer>>(); - errors = new HashMap<String, Map<String, Integer>>(); + warnings = new HashMap<String, List<ReportData>>(); + errors = new HashMap<String, List<ReportData>>(); } /** * @return the warnings */ - public Map<String, Map<String, Integer>> getWarnings() { + public Map<String, List<ReportData>> getWarnings() { return warnings; } /** * @return the errors */ - public Map<String, Map<String, Integer>> getErrors() { + public Map<String, List<ReportData>> getErrors() { return errors; } @Override public boolean importData(String content, AuthenticationResponse auth) { + this.warnings.clear(); + this.errors.clear(); + this.parser.reset(); try { boolean success = parser.parse(content); if (success) { @@ -110,20 +115,22 @@ writeMesswerte(auth, messwerte); } else { - Map<String, Integer> err = new HashMap<String, Integer>(); - err.put("no success", 660); - errors.put("parser", err); + List<ReportData> report = new ArrayList<ReportData>(); + report.add( new ReportData("parser", "no success", 660)); + errors.put("parser", report); return false; } } catch (LAFParserException e) { - Map<String, Integer> err = new HashMap<String, Integer>(); - err.put(e.getMessage(), 660); - errors.put("parser", err); + List<ReportData> report = new ArrayList<ReportData>(); + report.add(new ReportData("parser", e.getMessage(), 660)); + errors.put("parser", report); return false; } - Map<String, Map<String, Map<String, Integer>>> data = - new HashMap<String, Map<String,Map<String, Integer>>>(); + Map<String, Map<String, List<ReportData>>> data = + new HashMap<String, Map<String, List<ReportData>>>(); + this.warnings.putAll(this.parser.getWarnings()); + this.errors.putAll(this.parser.getErrors()); data.put("warnings", warnings); data.put("errors", errors); return true; @@ -140,16 +147,17 @@ messungValidator.validate(messung, false); messungRepository.create(messung); if (warn != null) { - warnings.put( - messung.getMessungsId().toString(), - warn); + for (String key : warn.keySet()) { + // warnings.put(messung.getProbeId(), + // new ReportData(key, "", warn.get(key))); + } } } catch (ValidationException e) { - errors.put(messung.getProbeId(), e.getErrors()); - warnings.put( - messung.getProbeId(), - e.getWarnings()); + //errors.put(messung.getProbeId(), e.getErrors()); + //warnings.put( + // messung.getProbeId(), + // e.getWarnings()); } } } @@ -165,16 +173,16 @@ messwertValidator.validate(messwert, false); Response r = messwertRepository.create(messwert); if (warn != null) { - warnings.put( - messwert.getProbeId(), - warn); + // warnings.put( + // messwert.getProbeId(), + // warn); } } catch (ValidationException e) { - errors.put(messwert.getProbeId(), e.getErrors()); - warnings.put( - messwert.getProbeId(), - e.getWarnings()); + //errors.put(messwert.getProbeId(), e.getErrors()); + //warnings.put( + // messwert.getProbeId(), + // e.getWarnings()); } } } @@ -193,7 +201,7 @@ err.put( kommentar.getProbeId() + " - " + kommentar.getkId(), 661); - errors.put("lkommentarp", err); + //errors.put("lkommentarp", err); } } } @@ -224,14 +232,14 @@ ortValidator.validate(ort, false); ortRepository.create(ort); if (warn != null) { - warnings.put(String.valueOf(ort.getOrtId()), warn); + // warnings.put(String.valueOf(ort.getOrtId()), warn); } } catch (ValidationException e) { - errors.put(String.valueOf(ort.getOrtId()), e.getErrors()); - warnings.put( - String.valueOf(ort.getOrtId()), - e.getWarnings()); + //errors.put(String.valueOf(ort.getOrtId()), e.getErrors()); + //warnings.put( + // String.valueOf(ort.getOrtId()), + // e.getWarnings()); } } } @@ -242,19 +250,19 @@ if (!authorized(probe, auth)) { Map<String, Integer> err = new HashMap<String, Integer>(); err.put("not authorized", 699); - errors.put(probe.getProbeId(), err); + //errors.put(probe.getProbeId(), err); continue; } try { Map<String, Integer> warn = probeValidator.validate(probe, false); if (warn != null) { - warnings.put(probe.getProbeId(), warn); + // warnings.put(probe.getProbeId(), warn); } } catch (ValidationException e) { - errors.put(probe.getProbeId(), e.getErrors()); - warnings.put(probe.getProbeId(), e.getWarnings()); + //errors.put(probe.getProbeId(), e.getErrors()); + //warnings.put(probe.getProbeId(), e.getWarnings()); continue; } persist(probe);
--- a/src/main/java/de/intevation/lada/data/importer/LAFParser.java Tue Aug 27 15:26:46 2013 +0200 +++ b/src/main/java/de/intevation/lada/data/importer/LAFParser.java Tue Aug 27 15:28:21 2013 +0200 @@ -1,7 +1,10 @@ package de.intevation.lada.data.importer; import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Map; import javax.inject.Inject; import javax.inject.Named; @@ -32,7 +35,12 @@ List<LKommentarP> probeKommentare; List<LKommentarM> messungKommentare; + private Map<String, List<ReportData>> warnings; + private Map<String, List<ReportData>> errors; + public LAFParser() { + this.warnings = new HashMap<String, List<ReportData>>(); + this.errors = new HashMap<String, List<ReportData>>(); this.setDryRun(false); //this.producer = new LAFProducer(); this.proben = new ArrayList<LProbe>(); @@ -64,11 +72,25 @@ if (nextPos > 0) { single = laf.substring(0, nextPos + 1); laf = laf.substring(nextPos + 1); - readAll(single); + try { + readAll(single); + } + catch (LAFParserException lpe) { + this.errors.putAll(producer.getErrors()); + this.producer.reset(); + continue; + } } else { - readAll(laf); - laf = ""; + try { + readAll(laf); + laf = ""; + } + catch (LAFParserException lpe) { + this.errors.putAll(producer.getErrors()); + laf = ""; + continue; + } } if (!this.dryRun) { proben.add(producer.getProbe()); @@ -235,4 +257,24 @@ public List<LKommentarM> getMessungKommentare() { return messungKommentare; } + + /** + * @return the warnings + */ + public Map<String, List<ReportData>> getWarnings() { + return warnings; + } + + /** + * @return the errors + */ + public Map<String, List<ReportData>> getErrors() { + return errors; + } + + public void reset() { + producer.reset(); + this.errors.clear(); + this.warnings.clear(); + } }
--- a/src/main/java/de/intevation/lada/data/importer/LAFProducer.java Tue Aug 27 15:26:46 2013 +0200 +++ b/src/main/java/de/intevation/lada/data/importer/LAFProducer.java Tue Aug 27 15:28:21 2013 +0200 @@ -1,7 +1,10 @@ package de.intevation.lada.data.importer; import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Map; import javax.inject.Inject; import javax.inject.Named; @@ -36,10 +39,15 @@ private List<EntryFormat> messungFormat; private List<EntryFormat> ortFormat; + private Map<String, List<ReportData>> warnings; + private Map<String, List<ReportData>> errors; + @Inject private AttributeMapper mapper; public LAFProducer() { + this.warnings = new HashMap<String, List<ReportData>>(); + this.errors = new HashMap<String, List<ReportData>>(); this.probe = new LProbe(); this.pKommentare = new ArrayList<LKommentarP>(); this.mKommentare = new ArrayList<LKommentarM>(); @@ -55,7 +63,8 @@ } @Override - public void addData(String key, Object values) { + public void addData(String key, Object values) + throws LAFParserException { String lKey = key.toLowerCase(); if(lKey.equals("probenkommentar")) { LKommentarP kommentar = new LKommentarP(); @@ -98,6 +107,10 @@ } else if (isValidProbe(lKey, values.toString())) { this.probe = mapper.addAttribute(lKey, values, this.probe); + if (this.probe == null) { + this.errors.put(values.toString(), mapper.getErrors()); + throw new LAFParserException(values.toString() + " exists"); + } } else if (isValidOrt(lKey, values.toString())) { this.ort = mapper.addAttribute(lKey, values, this.ort); @@ -169,6 +182,8 @@ @Override public void reset() { + this.errors.clear(); + this.warnings.clear(); this.probe = new LProbe(); this.messungen.clear(); this.messung = null; @@ -177,6 +192,7 @@ this.messwerte.clear(); this.mKommentare.clear(); this.pKommentare.clear(); + mapper.reset(); } public void newMessung() { @@ -197,4 +213,18 @@ this.ort = new LOrt(); this.ort.setProbeId(this.probe.getProbeId()); } + + /** + * @return the warnings + */ + public Map<String, List<ReportData>> getWarnings() { + return this.warnings; + } + + /** + * @return the errors + */ + public Map<String, List<ReportData>> getErrors() { + return this.errors; + } }
--- a/src/main/java/de/intevation/lada/data/importer/Producer.java Tue Aug 27 15:26:46 2013 +0200 +++ b/src/main/java/de/intevation/lada/data/importer/Producer.java Tue Aug 27 15:28:21 2013 +0200 @@ -1,6 +1,7 @@ package de.intevation.lada.data.importer; import java.util.List; +import java.util.Map; import de.intevation.lada.model.LKommentarM; import de.intevation.lada.model.LKommentarP; @@ -12,7 +13,8 @@ public interface Producer { - public void addData(String key, Object values); + public void addData(String key, Object values) + throws LAFParserException; public LProbe getProbe(); public List<LMessung> getMessungen(); public List<LOrt> getOrte(); @@ -22,4 +24,6 @@ public void reset(); public void newMessung(); public void newOrt(); + public Map<String, List<ReportData>> getErrors(); + public Map<String, List<ReportData>> getWarnings(); }
--- a/src/main/java/de/intevation/lada/rest/LAFImportService.java Tue Aug 27 15:26:46 2013 +0200 +++ b/src/main/java/de/intevation/lada/rest/LAFImportService.java Tue Aug 27 15:28:21 2013 +0200 @@ -88,12 +88,10 @@ } boolean success = importer.importData(content, auth); - List<Object> respData = new LinkedList<Object>(); - respData.add(importer.getErrors()); - respData.add(importer.getWarnings()); - Map<String, String> fileData = new HashMap<String, String>(); - fileData.put("filename", name); - respData.add(fileData); + Map<String, Object> respData = new HashMap<String,Object>(); + respData.put("errors", importer.getErrors()); + respData.put("warnings", importer.getWarnings()); + respData.put("filename", name); int code = 200; if (!success) { code = 660;