Mercurial > lada > lada-server
changeset 310:821557a17e5e
First version of the LAF importer.
The importer module currently only runs with the test application (comming in the next commit)!
* LAF Parser:
- Uses a small implementation of a state machine.
- Extracts the keys with its value or multi value.
- Uses the producer interface to generate objects.
* Attribute mapper:
- Maps the attributes defined in the configuration file to object attributes.
- Generates objects from multi value attributes.
* LAF format:
- Reads the config file
* LAF importer:
- Implemetation of the importer interface for LAF format.
| author | Raimund Renkert <rrenkert@intevation.de> |
|---|---|
| date | Tue, 20 Aug 2013 16:13:17 +0200 |
| parents | ae4bf396bd3b |
| children | eeb5d3a5e194 |
| files | src/main/java/de/intevation/lada/data/importer/AttributeMapper.java src/main/java/de/intevation/lada/data/importer/EntryFormat.java src/main/java/de/intevation/lada/data/importer/LAFFormat.java src/main/java/de/intevation/lada/data/importer/LAFImporter.java src/main/java/de/intevation/lada/data/importer/LAFParser.java src/main/java/de/intevation/lada/data/importer/LAFParserException.java src/main/java/de/intevation/lada/data/importer/LAFProducer.java |
| diffstat | 7 files changed, 730 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/intevation/lada/data/importer/AttributeMapper.java Tue Aug 20 16:13:17 2013 +0200 @@ -0,0 +1,238 @@ +package de.intevation.lada.data.importer; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.inject.Inject; +import javax.persistence.EntityManager; + +import de.intevation.lada.model.LKommentarM; +import de.intevation.lada.model.LKommentarP; +import de.intevation.lada.model.LMessung; +import de.intevation.lada.model.LMessungId; +import de.intevation.lada.model.LMesswert; +import de.intevation.lada.model.LOrt; +import de.intevation.lada.model.LProbe; + + +public class AttributeMapper +{ + @Inject + private EntityManager em; + + public LProbe addAttribute(String key, Object value, LProbe probe) { + DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm"); + if ("datenbasis_s".equals(key)) { + Integer v = Integer.valueOf(value.toString()); + probe.setDatenbasisId(v); + } + else if ("probe_id".equals(key)) { + probe.setProbeId(value.toString()); + } + else if ("hauptprobennummer".equals(key)) { + probe.setHauptprobenNr(value.toString()); + } + else if ("mpr_id".equals(key)) { + Integer v = Integer.valueOf(value.toString()); + probe.setMprId(v); + } + else if ("messprogramm_land".equals(key)) { + probe.setMplId(value.toString()); + } + else if ("messstelle".equals(key)) { + probe.setMstId(value.toString()); + } + else if ("betriebsart".equals(key)) { + probe.setBaId(value.toString()); + } + else if ("soll_datum_uhrzeit_a".equals(key)) { + try { + Date d = format.parse(value.toString()); + probe.setSolldatumBeginn(d); + } + catch (ParseException e) { + //TODO handle warning. + } + } + else if ("soll_datum_uhrzeit_e".equals(key)) { + try { + Date d = format.parse(value.toString()); + probe.setSolldatumEnde(d); + } + catch (ParseException e) { + //TODO handle warning. + } + } + else if ("probenahme_datum_uhrzeit_a".equals(key)) { + try { + Date d = format.parse(value.toString()); + probe.setProbeentnahmeBeginn(d); + } + catch (ParseException e) { + //TODO handle warning. + } + } + else if ("probenahme_datum_uhrzeit_e".equals(key)) { + try { + Date d = format.parse(value.toString()); + probe.setProbeentnahmeEnde(d); + } + catch (ParseException e) { + //TODO handle warning. + } + } + else if ("umweltbereich_s".equals(key)) { + probe.setUmwId(value.toString()); + } + else if ("deskriptoren".equals(key)) { + probe.setMediaDesk(value.toString()); + } + else if ("testdaten".equals(key)) { + if (!value.toString().equals("0")) { + probe.setTest(true); + } + else { + probe.setTest(false); + } + } + return probe; + } + + public LKommentarP addAttribute( + String key, + Object values, + LKommentarP kommentar + ) { + DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm"); + String v = values.toString(); + String erzeuger = v.substring(1, 5); + String date = v.substring(8, 21); + Date d; + try { + d = format.parse(date); + kommentar.setKDatum(d); + } + catch (ParseException e) { + //TODO: handle warning. + } + String text = v.substring(23, v.length() -1); + kommentar.setErzeuger(erzeuger); + kommentar.setKText(text); + return kommentar; + } + + public LKommentarM addAttribute( + String key, + Object values, + LKommentarM kommentar + ) { + DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm"); + String v = values.toString(); + String erzeuger = v.substring(1, 5); + String date = v.substring(8, 21); + Date d; + try { + d = format.parse(date); + kommentar.setKDatum(d); + } + catch (ParseException e) { + //TODO: handle warning. + } + String text = v.substring(23, v.length() -1); + kommentar.setErzeuger(erzeuger); + kommentar.setKText(text); + return kommentar; + } + + public LMessung addAttribute( + String key, + Object values, + LMessung messung + ) { + DateFormat format = new SimpleDateFormat("yyyyMMDD hhmm"); + if ("messungs_id".equals(key)) { + LMessungId id = messung.getId(); + Integer v = Integer.valueOf(values.toString()); + id.setMessungsId(v); + messung.setId(id); + } + else if ("nebenprobennummer".equals(key)) { + messung.setNebenprobenNr(values.toString()); + } + else if ("mess_datum_uhrzeit".equals(key)) { + try { + Date d = format.parse(values.toString()); + messung.setMesszeitpunkt(d); + } + catch (ParseException e) { + //TODO: handle warnings. + } + } + else if ("messzeit_sekunden".equals(key)) { + Integer i = Integer.valueOf(values.toString()); + messung.setMessdauer(i); + } + else if ("messmethode_s".equals(key)) { + messung.setMmtId(values.toString()); + } + else if ("bearbeitungsstatus".equals(key)) { + //ignored.!? + } + else if ("erfassung_abgeschlossen".equals(key)) { + if(!values.toString().equals("0")) { + messung.setFertig(true); + } + else { + messung.setFertig(false); + } + } + return messung; + } + + public LMesswert addAttribute( + String key, + Object values, + LMesswert messwert + ) { + Pattern p = Pattern.compile( + "(\".+\")( .+ )(\".+\")( .*)( .{1,12})( .{1,9})(.{0,9})(.{0,3})"); + //TODO Does not perfectly match... Use better matching for floats. + Matcher m = p.matcher(values.toString()); + if (m.matches()) { + String messgroesse = m.group(1); + String wert = m.group(2); + String einheit = m.group(3); + if (wert.startsWith(" <")) { + wert = wert.substring(2); + messwert.setGrenzwertueberschreitung(false); + } + else if (wert.startsWith(" >")) { + wert = wert.substring(2); + messwert.setGrenzwertueberschreitung(true); + } + float fWert = Float.valueOf(wert); + messwert.setMesswert(fWert); + } + //TODO: Match the other values. + return messwert; + } + + public LOrt addAttribute( + String key, + Object values, + LOrt ort + ) { + if ("ort_id".equals(key)) { + Integer v = Integer.valueOf(values.toString()); + ort.setOrtId(v); + } + else if ("ort_typ".equals(key)) { + ort.setOrtsTyp(values.toString()); + } + return ort; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/intevation/lada/data/importer/EntryFormat.java Tue Aug 20 16:13:17 2013 +0200 @@ -0,0 +1,38 @@ +package de.intevation.lada.data.importer; + +import java.util.regex.Pattern; + + +public class EntryFormat +{ + private String key; + private Pattern pattern; + private Object defaultValue; + + public EntryFormat() { + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public Pattern getPattern() { + return pattern; + } + + public void setPattern(Pattern pattern) { + this.pattern = pattern; + } + + public Object getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(Object defaultValue) { + this.defaultValue = defaultValue; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/intevation/lada/data/importer/LAFFormat.java Tue Aug 20 16:13:17 2013 +0200 @@ -0,0 +1,62 @@ +package de.intevation.lada.data.importer; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Pattern; + +import javax.ejb.Singleton; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + + +public class LAFFormat +{ + private JSONObject fileContent; + + public boolean readConfigFile(String fileName) { + try { + byte[] encoded = Files.readAllBytes(Paths.get(fileName)); + Charset encoding = Charset.defaultCharset(); + String content = + encoding.decode(ByteBuffer.wrap(encoded)).toString(); + fileContent = new JSONObject(content); + return true; + } + catch (IOException ioe) { + return false; + } + catch (JSONException je) { + return false; + } + } + + public List<EntryFormat> getFormat(String dataType) { + List<EntryFormat> formats = new LinkedList<EntryFormat>(); + try { + JSONArray block = fileContent.getJSONArray(dataType); + for (int i = 0; i < block.length(); i++) { + JSONObject jEntry = block.getJSONObject(i); + EntryFormat entry = new EntryFormat(); + entry.setKey(jEntry.getString("key")); + Pattern pattern = + Pattern.compile( + jEntry.getString("regex"), + Pattern.MULTILINE); + entry.setPattern(pattern); + entry.setDefaultValue(jEntry.get("default")); + formats.add(entry); + } + return formats; + } + catch (JSONException e) { + return null; + } + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/intevation/lada/data/importer/LAFImporter.java Tue Aug 20 16:13:17 2013 +0200 @@ -0,0 +1,16 @@ +package de.intevation.lada.data.importer; + +import javax.inject.Named; + +import de.intevation.lada.auth.AuthenticationResponse; +import de.intevation.lada.rest.Response; + +@Named("lafimporter") +public class LAFImporter +implements Importer +{ + @Override + public Response importData(String content, AuthenticationResponse auth) { + return new Response(true, 200, null); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/intevation/lada/data/importer/LAFParser.java Tue Aug 20 16:13:17 2013 +0200 @@ -0,0 +1,181 @@ +package de.intevation.lada.data.importer; + +import java.util.ArrayList; +import java.util.List; + +import javax.inject.Inject; +import javax.inject.Named; + +import de.intevation.lada.model.LKommentarM; +import de.intevation.lada.model.LKommentarP; +import de.intevation.lada.model.LMessung; +import de.intevation.lada.model.LMesswert; +import de.intevation.lada.model.LOrt; +import de.intevation.lada.model.LProbe; + + +public class LAFParser { + + private static final String PROBE_NEXT = "\n%PROBE%"; + + private boolean dryRun; + + //@Inject + //@Named("lafproducer") + private Producer producer; + + List<LProbe> proben; + List<LMessung> messungen; + List<LOrt> orte; + List<LMesswert> messwerte; + List<LKommentarP> probeKommentare; + List<LKommentarM> messungKommentare; + + public LAFParser() { + this.setDryRun(false); + this.producer = new LAFProducer(); + this.proben = new ArrayList<LProbe>(); + this.messungen = new ArrayList<LMessung>(); + this.orte = new ArrayList<LOrt>(); + this.messwerte = new ArrayList<LMesswert>(); + this.probeKommentare = new ArrayList<LKommentarP>(); + this.messungKommentare = new ArrayList<LKommentarM>(); + } + + public boolean parse(String laf) + throws LAFParserException + { + boolean parsed = false; + while (laf.startsWith("%PROBE%\n")) { + parsed = true; + int nextPos = laf.indexOf(PROBE_NEXT); + String single = ""; + if (nextPos > 0) { + single = laf.substring(0, nextPos + 1); + laf = laf.substring(nextPos + 1); + readAll(single); + } + else { + readAll(laf); + laf = ""; + } + if (!this.dryRun) { + proben.add(producer.getProbe()); + messungen.addAll(producer.getMessungen()); + orte.addAll(producer.getOrte()); + messwerte.addAll(producer.getMesswerte()); + probeKommentare.addAll(producer.getProbenKommentare()); + messungKommentare.addAll(producer.getMessungsKommentare()); + producer.reset(); + } + } + if (!parsed) { + throw new LAFParserException("No %PROBE% at the begining."); + } + return parsed; + } + + private void readAll(String content) + throws LAFParserException + { + boolean key = false; + boolean value = false; + boolean header = false; + boolean white = false; + boolean string = false; + boolean multiValue = false; + String keyString = ""; + String valueString = ""; + String headerString = ""; + for (int i = 0; i < content.length(); i++) { + char current = content.charAt(i); + + if ((current == '"' || (current == ' ' && !string)) && + value && + i < content.length() - 1 && + (content.charAt(i + 1) != '\n' && + content.charAt(i + 1) != '\r')) { + multiValue = true; + } + + if (current == '"' && !string) { + string = true; + } + else if (current == '"' && string) { + string = false; + } + + if (current == ' ' && !value) { + key = false; + white = true; + continue; + } + else if (current != ' ' && white) { + value = true; + white = false; + } + else if (current == '%' && !header && !value) { + headerString = ""; + + key = false; + header = true; + } + else if ((current == '\n' || current == '\r') && header) { + header = false; + key = true; + if (!dryRun) { + if (headerString.contains("MESSUNG")) { + producer.newMessung(); + } + if (headerString.contains("ORT")) { + producer.newOrt(); + } + } + continue; + } + else if (current == '"' && !value) { + value = true; + } + else if ((current == '\n' || current == '\r') && value && !string) { + if (!multiValue && valueString.startsWith("\"")) { + valueString = + valueString.substring(1, valueString.length() - 1); + } + value = false; + multiValue = false; + key = true; + if (!this.dryRun) { + producer.addData(keyString, valueString); + } + keyString = ""; + valueString = ""; + continue; + } + else if ((current == '\n' || current == '\r') && key) { + throw new LAFParserException("No value for key: " + keyString); + } + + if (key) { + keyString += current; + } + else if (value) { + valueString += current; + } + else if (header) { + headerString += current; + } + } + if (!dryRun) { + this.producer.newMessung(); + this.producer.newOrt(); + } + } + + public boolean isDryRun() { + return dryRun; + } + + public void setDryRun(boolean dryRun) { + this.dryRun = dryRun; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/intevation/lada/data/importer/LAFParserException.java Tue Aug 20 16:13:17 2013 +0200 @@ -0,0 +1,11 @@ +package de.intevation.lada.data.importer; + + +public class LAFParserException +extends Exception +{ + + public LAFParserException(String message) { + super(message); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/intevation/lada/data/importer/LAFProducer.java Tue Aug 20 16:13:17 2013 +0200 @@ -0,0 +1,184 @@ +package de.intevation.lada.data.importer; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import javax.inject.Named; + +import de.intevation.lada.model.LKommentarM; +import de.intevation.lada.model.LKommentarP; +import de.intevation.lada.model.LMessung; +import de.intevation.lada.model.LMessungId; +import de.intevation.lada.model.LMesswert; +import de.intevation.lada.model.LMesswertId; +import de.intevation.lada.model.LOrt; +import de.intevation.lada.model.LProbe; + +@Named("lafproducer") +public class LAFProducer +implements Producer +{ + + private LProbe probe; + private LMessung messung; + private LOrt ort; + + private List<LKommentarP> pKommentare; + private List<LKommentarM> mKommentare; + private List<LMessung> messungen; + private List<LOrt> orte; + private List<LMesswert> messwerte; + + private List<EntryFormat> probenFormat; + private List<EntryFormat> messungFormat; + private List<EntryFormat> ortFormat; + + private AttributeMapper mapper; + + public LAFProducer() { + this.mapper = new AttributeMapper(); + this.probe = new LProbe(); + this.pKommentare = new ArrayList<LKommentarP>(); + this.mKommentare = new ArrayList<LKommentarM>(); + this.messungen = new ArrayList<LMessung>(); + this.orte = new ArrayList<LOrt>(); + this.messwerte = new ArrayList<LMesswert>(); + String fileName = System.getProperty("de_intevation_lada_import"); + LAFFormat format = new LAFFormat(); + format.readConfigFile(fileName); + probenFormat = format.getFormat("probe"); + messungFormat = format.getFormat("messung"); + ortFormat = format.getFormat("ort"); + } + + @Override + public void addData(String key, Object values) { + String lKey = key.toLowerCase(); + if(lKey.equals("probenkommentar")) { + this.pKommentare.add( + mapper.addAttribute(lKey, values, new LKommentarP())); + } + else if (lKey.equals("kommentar")) { + this.mKommentare.add( + mapper.addAttribute(lKey, values, new LKommentarM())); + } + else if (lKey.equals("probenzusatzbeschreibung")) { + //TODO: implement this! + } + else if (lKey.equals("pzb_s")) { + //TODO: implement this! + } + else if (lKey.equals("messwert")) { + LMesswertId id = new LMesswertId(); + id.setProbeId(this.probe.getProbeId()); + id.setMessungsId(this.messung.getId().getMessungsId()); + LMesswert m = new LMesswert(); + m.setId(id); + this.messwerte.add( + mapper.addAttribute(lKey, values, m)); + } + else if (isValidMessung(lKey, values.toString())) { + this.messung = mapper.addAttribute(lKey, values, this.messung); + } + else if (isValidProbe(lKey, values.toString())) { + this.probe = mapper.addAttribute(lKey, values, this.probe); + } + else if (isValidOrt(lKey, values.toString())) { + this.ort = mapper.addAttribute(lKey, values, this.ort); + } + } + + private boolean isValidOrt(String key, String value) { + for (EntryFormat ef: ortFormat) { + if (ef.getKey().equals(key.toLowerCase())) { + if (ef.getPattern().matcher(value).matches()) { + return true; + } + } + } + return false; + } + + private boolean isValidMessung(String key, String value) { + for (EntryFormat ef: messungFormat) { + if (ef.getKey().equals(key.toLowerCase())) { + if (ef.getPattern().matcher(value).matches()) { + return true; + } + } + } + return false; + } + + private boolean isValidProbe(String key, String value) { + for (EntryFormat ef: probenFormat) { + if (ef.getKey().equals(key.toLowerCase())) { + if (ef.getPattern().matcher(value).matches()) { + return true; + } + } + } + return false; + } + + @Override + public LProbe getProbe() { + return this.probe; + } + + @Override + public List<LMessung> getMessungen() { + return this.messungen; + } + + @Override + public List<LOrt> getOrte() { + return this.orte; + } + + @Override + public List<LKommentarP> getProbenKommentare() { + return this.pKommentare; + } + + @Override + public List<LKommentarM> getMessungsKommentare() { + return this.mKommentare; + } + + @Override + public List<LMesswert> getMesswerte() { + return this.messwerte; + } + + @Override + public void reset() { + this.probe = new LProbe(); + this.messungen.clear(); + this.messung = null; + this.orte.clear(); + this.ort = null; + this.messwerte.clear(); + this.mKommentare.clear(); + this.pKommentare.clear(); + } + + public void newMessung() { + if (this.messung != null) { + this.messungen.add(this.messung); + } + LMessungId id = new LMessungId(); + id.setProbeId(this.probe.getProbeId()); + this.messung = new LMessung(); + this.messung.setId(id); + } + + public void newOrt() { + if (this.ort != null) { + this.orte.add(this.ort); + } + this.ort = new LOrt(); + this.ort.setProbeId(this.probe.getProbeId()); + } +}