changes

src/osmfixer/CMakeLists.txt

@@ -20,7 +20,7 @@ add_custom_command(
 
 add_custom_command(
 	OUTPUT build.h
-	COMMAND cd ${PROJECT_BINARY_DIR}/../web/ && cat leaflet-heat.js script.js > build.js && xxd -i build.js > ${CMAKE_CURRENT_BINARY_DIR}/build.h && rm build.js
+	COMMAND cd ${PROJECT_BINARY_DIR}/../web/ && cat leaflet.js leaflet-heat.js script.js > merged.js && java -jar closurec/compiler.jar -O SIMPLE leaflet.js leaflet-heat.js script.js > build.js && xxd -i build.js > ${CMAKE_CURRENT_BINARY_DIR}/build.h && rm build.js
 	DEPENDS "${PROJECT_BINARY_DIR}/../web/script.js"
 	DEPENDS "${PROJECT_BINARY_DIR}/../web/leaflet-heat.js"
 	VERBATIM

src/osmfixer/FixerMain.cpp

@@ -56,6 +56,7 @@ int main(int argc, char** argv) {
   std::vector<std::pair<IdRange, StatIdx>> idx(inputPaths.size());
   IdRange idRange{0, 0, 0, 0, 0, 0};
   for (size_t i = 0; i < inputPaths.size(); i++) {
+    LOG(INFO) << "Processing " << inputPaths[i];
     idx[i].first = idRange;
     if (i > 0) {
       idx[i].first.sidStart = idx[i - 1].first.sidEnd + 1;
@@ -70,7 +71,7 @@ int main(int argc, char** argv) {
   }
 
   LOG(INFO) << "Building search index...";
-  osmfixer::SearchIdx searchIdx(idx.front().second);
+  osmfixer::SearchIdx searchIdx(idx);
 
   LOG(INFO) << "Starting server...";
   StatServer serv(idx, searchIdx);

src/osmfixer/index/SearchIdx.cpp

@@ -14,57 +14,64 @@
 #include "osmfixer/index/SearchIdx.h"
 
 using osmfixer::SearchIdx;
-using osmfixer::TupleList;
 using osmfixer::TripleList;
+using osmfixer::TupleList;
 
 // _____________________________________________________________________________
-void SearchIdx::build() {
+void SearchIdx::build(std::vector<std::pair<IdRange, StatIdx>>& idxs) {
   _qGramIndex.clear();
 
   size_t nameid = 0;
 
   std::map<std::wstring, size_t> tokenIds;
 
-  for (size_t gid = 0; gid < _stats.getGroups().size(); gid++) {
-		auto group = _stats.getGroup(gid);
-
-		// dont index empty groups
-		if (group->stations.size() == 0) continue;
-		if (group->polyStations.size() == 0) continue;
-
-    for (const auto& name : _stats.getGroup(gid)->uniqueNames) {
-      // use wstring to get UTF-8 chars right
-      std::wstring wname =
-          std::wstring_convert<std::codecvt_utf8<wchar_t>>().from_bytes(name);
-
-      _nameToGroup[nameid] = gid;
-			_names.push_back(name);
-
-      for (const auto& token : tokenize(wname)) {
-        if (tokenIds.count(token)) {
-          if (_inv[tokenIds.find(token)->second].size() == 0 ||
-              _inv[tokenIds.find(token)->second].back().first != nameid) {
-            // only use a token once per station
-            _inv[tokenIds.find(token)->second].push_back({nameid, 1});
+  for (const auto& idx : idxs) {
+    for (size_t gid = 0; gid < idx.second.getGroups().size(); gid++) {
+      auto group = idx.second.getGroup(gid);
+
+      // slightly prefer larger stations
+      double groupScore = 1.0 + log(group->stations.size()) / 20;
+
+      // dont index empty groups
+      if (group->stations.size() == 0) continue;
+      if (group->polyStations.size() == 0) continue;
+
+      for (const auto& namep : group->uniqueNames) {
+        // use wstring to get UTF-8 chars right
+        const auto& name = namep.second;
+        std::wstring wname =
+            std::wstring_convert<std::codecvt_utf8<wchar_t>>().from_bytes(name);
+
+        _nameToGroup[nameid] = idx.first.gidStart + gid;
+        _names.push_back(name);
+
+        for (const auto& token : tokenize(wname)) {
+          if (tokenIds.count(token)) {
+            if (_inv[tokenIds.find(token)->second].size() == 0 ||
+                _inv[tokenIds.find(token)->second].back().first != nameid) {
+              // only use a token once per station
+              _inv[tokenIds.find(token)->second].push_back(
+                  {nameid, groupScore});
+            }
+          } else {
+            tokenIds[token] = _tokens.size();
+            _tokens.push_back(token);
+            _inv.push_back(TupleList());
+            _inv.back().push_back({nameid, groupScore});
           }
-        } else {
-          tokenIds[token] = _tokens.size();
-          _tokens.push_back(token);
-          _inv.push_back(TupleList());
-          _inv.back().push_back({nameid, 1});
-        }
 
-        size_t tokenId = tokenIds.find(token)->second;
-        for (const auto& qGram : getQGrams(token)) {
-          if (_qGramIndex[qGram].size() &&
-              _qGramIndex[qGram].back().first == tokenId) {
-            _qGramIndex[qGram].back().second++;
-          } else {
-            _qGramIndex[qGram].push_back({tokenId, 1});
+          size_t tokenId = tokenIds.find(token)->second;
+          for (const auto& qGram : getQGrams(token)) {
+            if (_qGramIndex[qGram].size() &&
+                _qGramIndex[qGram].back().first == tokenId) {
+              _qGramIndex[qGram].back().second++;
+            } else {
+              _qGramIndex[qGram].push_back({tokenId, 1});
+            }
           }
         }
+        nameid++;
       }
-      nameid++;
     }
   }
 
@@ -101,7 +108,7 @@ std::vector<std::wstring> SearchIdx::tokenize(const std::wstring& str) {
   std::vector<std::wstring> ret;
   std::wstring cur;
 
-  const wchar_t* seps = L"_-?'\"|!@#$%^&*()_+}|><.,\\";
+  const wchar_t* seps = L"_-?'\"|!@#$%^&*()_+{}[]|<>.:,\\/";
 
   for (size_t i = 0; i < str.size(); i++) {
     if (std::iswspace(str[i]) || wcschr(seps, str[i])) {
@@ -147,42 +154,43 @@ TripleList SearchIdx::find(const std::string& qry) const {
     double delta = token.size() / 4.0;
     auto res = tokenFind(token);
 
-    std::partial_sort(res.begin(), res.begin() + std::min<size_t>(100, res.size()), res.end(), resComp);
+    std::partial_sort(res.begin(),
+                      res.begin() + std::min<size_t>(100, res.size()),
+                      res.end(), resComp);
 
     std::map<size_t, double> bests;
     std::map<size_t, size_t> bestToken;
 
-		size_t TOPK = 100;
+    size_t TOPK = 100;
 
-		// res contains the 100 best token matches
+    // res contains the 100 best token matches
     for (size_t i = 0; i < res.size() && i < TOPK; i++) {
       for (size_t j = 0; j < _inv[res[i].first].size(); j++) {
+        double score =
+            _inv[res[i].first][j].second * (delta / (1.0 + res[i].second));
 
-				double score = _inv[res[i].first][j].second * (delta / (1.0 + res[i].second));
-
-				if (score > bests[_inv[res[i].first][j].first]) {
-						bests[_inv[res[i].first][j].first] = score;
-						bestToken[_inv[res[i].first][j].first] = res[i].first;
-				}
+        if (score > bests[_inv[res[i].first][j].first]) {
+          bests[_inv[res[i].first][j].first] = score;
+          bestToken[_inv[res[i].first][j].first] = res[i].first;
+        }
       }
     }
 
     for (size_t i = 0; i < res.size() && i < TOPK; i++) {
-			// inv[res[i]] contains all the names the token res[i] occurs in
+      // inv[res[i]] contains all the names the token res[i] occurs in
 
       lists.push_back(_inv[res[i].first]);
 
       // give them a score based on their PED
-			for (size_t j = 0; j < lists.back().size(); j++) {
-				double score = lists.back()[j].second *
-                                 (delta / (1.0 + res[i].second));
-				// best is the token for this name that matched best for the input token
-				size_t best = bestToken[lists.back()[j].first];
-
-				// if it is not this token, we dont count it
-				if (res[i].first != best) score = 0;
+      for (size_t j = 0; j < lists.back().size(); j++) {
+        double score = lists.back()[j].second * (delta / (1.0 + res[i].second));
+        // best is the token for this name that matched best for the input token
+        size_t best = bestToken[lists.back()[j].first];
+
+        // if it is not this token, we dont count it
+        if (res[i].first != best) score = 0;
         lists.back()[j].second = score;
-			}
+      }
     }
   }
 
@@ -201,7 +209,8 @@ TripleList SearchIdx::find(const std::string& qry) const {
 
   TripleList fin;
   for (const auto& r : fr) {
-    if (fin.size() == 0 || fin.back().first.first != r.first.first) fin.push_back(r);
+    if (fin.size() == 0 || fin.back().first.first != r.first.first)
+      fin.push_back(r);
   }
 
   std::partial_sort(fin.begin(), fin.begin() + std::min<size_t>(fin.size(), 10),

src/osmfixer/index/SearchIdx.h

@@ -29,15 +29,20 @@ inline bool resCompInv(const Triple& lh, const Triple& rh) {
 
 inline bool resGidCompInv(const Triple& lh, const Triple& rh) {
   if (lh.first.first < rh.first.first) return true;
-  if (lh.first.first == rh.first.first) return lh.second > rh.second;
+  if (lh.first.first == rh.first.first) {
+    // if the scores are equal, order by the name id to prefer the original
+    // name id, which is lowest!
+    if (fabs(lh.second - rh.second) < 0.01) return lh.first.second < rh.first.second;
+    return lh.second > rh.second;
+  }
   return false;
 }
 
 class SearchIdx {
  public:
-  explicit SearchIdx(const StatIdx& sidx) : _stats(sidx) { build(); }
+  explicit SearchIdx(std::vector<std::pair<IdRange, StatIdx>>& idxs) { build(idxs); }
 
-  void build();
+  void build(std::vector<std::pair<IdRange, StatIdx>>& idxs);
 
   TripleList find(const std::string& qry) const;
 
@@ -54,8 +59,6 @@ class SearchIdx {
   TupleList tokenFind(const std::wstring& prefix) const;
   static TupleList lmerge(const std::vector<const TupleList*>& lists);
 
-  const StatIdx& _stats;
-
   Index _qGramIndex;
 
   std::unordered_map<size_t, size_t> _nameToGroup;

src/osmfixer/index/StatIdx.cpp

@@ -210,7 +210,6 @@ void StatIdx::addGroup(size_t osmid, const OsmAttrs& attrs) {
 // _____________________________________________________________________________
 void StatIdx::initGroups() {
   for (size_t i = 0; i < _stations.size(); i++) {
-
     // this should be ensured by the input file
     assert(_stations[i].origGroup < _groups.size());
     _groups[_stations[i].origGroup].polyStations.push_back(i);
@@ -238,24 +237,26 @@ void StatIdx::initGroups() {
         if (usedGroupNames.count(name)) continue;
         usedGroupNames.insert(name);
 
-        _groups[gid].uniqueNames.push_back(name);
+        // relation names are preferred
+        _groups[gid].uniqueNames.push_back({nameAttrRel(np.first) + 3, name});
       }
     }
 
-    for (size_t sid :_groups[gid].stations) {
+    for (size_t sid : _groups[gid].stations) {
       for (const auto& np : _stations[sid].attrs) {
         for (const auto& name : np.second) {
           // use every name only once to keep the index small
           if (usedGroupNames.count(name)) continue;
           usedGroupNames.insert(name);
 
-          _groups[gid].uniqueNames.push_back(name);
+          _groups[gid].uniqueNames.push_back({nameAttrRel(np.first), name});
         }
       }
     }
 
     // take the longest name as the identifier
-    std::sort(_groups[gid].uniqueNames.begin(), _groups[gid].uniqueNames.end(), byLen);
+    std::sort(_groups[gid].uniqueNames.begin(), _groups[gid].uniqueNames.end(),
+              byAttrScore);
   }
 
   // build hull polygon
@@ -284,7 +285,6 @@ util::geo::Polygon<double> StatIdx::hull(
   util::geo::MultiPoint<double> mp;
   util::geo::Polygon<double> ret;
   for (const auto& geom : imp) {
-    // double rad = 11.0;
     int n = 20;
     for (int i = 0; i < n; i++) {
       double x = rad * cos((2.0 * M_PI / static_cast<double>(n)) *
@@ -311,7 +311,10 @@ void StatIdx::initIndex() {
                                                               _bbox, false);
 
   for (size_t i = 0; i < _stations.size(); i++) _sgrid.add(_stations[i].pos, i);
-  for (size_t i = 0; i < _groups.size(); i++) _ggrid.add(_groups[i].poly, i);
+  for (size_t i = 0; i < _groups.size(); i++) {
+    if (_groups[i].polyStations.size() == 1 && _groups[i].osmid < 2) continue;
+    _ggrid.add(_groups[i].poly, i);
+  }
   for (size_t i = 0; i < _suggestions.size(); i++)
     _suggrid.add(_suggestions[i].arrow, i);
 
@@ -566,7 +569,7 @@ void StatIdx::initSuggestions() {
   for (size_t i = 0; i < _groups.size(); i++) {
     auto& group = _groups[i];
 
-    if (group.attrs.size() == 0) {
+    if (group.attrs.count("name") == 0) {
       Suggestion sug;
       sug.station = i;
       sug.type = 7;
@@ -616,7 +619,7 @@ void StatIdx::initSuggestions() {
   for (size_t i = 0; i < _stations.size(); i++) {
     auto& stat = _stations[i];
 
-    if (stat.attrs.size() == 0) {
+    if (stat.attrs.count("name") == 0 && _groups[stat.group].attrs.count("name") == 0) {
       Suggestion sug;
       sug.station = i;
       sug.type = 7;
@@ -671,7 +674,8 @@ void StatIdx::initSuggestions() {
       sug.arrow = util::geo::DLine{centroid, centroid};
 
       if (getGroup(stat.origGroup)->osmid < 2) {
-        if (getGroup(stat.group)->osmid == 1) {
+        if (getGroup(stat.group)->osmid == 1 &&
+            getGroup(stat.group)->stations.size() > 1) {
           // move orphan into new group
           sug.type = 1;
           sug.target_gid = stat.group;
@@ -739,7 +743,6 @@ void StatIdx::initSuggestions() {
         }
       }
     }
-
   }
 }
 
@@ -799,6 +802,21 @@ const Group* StatIdx::getGroup(size_t id) const {
 }
 
 // _____________________________________________________________________________
+int StatIdx::nameAttrRel(const std::string& attr) const {
+  if (attr == "uic_name") return 5;
+  if (attr == "ref_name") return 5;
+  if (attr == "nat_name") return 4;
+  if (attr == "official_name") return 3;
+  if (attr == "gtfs_name") return 3;
+  if (attr == "int_name") return 2;
+  if (attr == "name") return 1;
+
+  if (attr == "alt_name") return -1;
+
+  return 0;
+}
+
+// _____________________________________________________________________________
 const Suggestion* StatIdx::getSuggestion(size_t id) const {
   if (id >= _suggestions.size()) return 0;
   return &_suggestions[id];

src/osmfixer/index/StatIdx.h

@@ -15,10 +15,15 @@ namespace osmfixer {
 
 typedef std::map<std::string, std::vector<std::string>> OsmAttrs;
 
-inline bool byLen(const std::string& lh, const std::string& rh) {
-  return lh.size() > rh.size();
+inline bool byAttrScore(const std::pair<int, std::string>& lh,
+                  const std::pair<int, std::string>& rh) {
+  return lh.first > rh.first;
 }
 
+struct IdRange {
+  size_t sidStart, sidEnd, gidStart, gidEnd, suggIdStart, suggIdEnd;
+};
+
 struct AttrErr {
   std::string attr;
   std::string otherAttr;
@@ -60,7 +65,7 @@ struct Group {
   std::vector<size_t> polyStations;
   std::vector<AttrErr> attrErrs;
   std::vector<size_t> suggestions;
-  std::vector<std::string> uniqueNames;
+  std::vector<std::pair<int, std::string>> uniqueNames;
 };
 
 struct Cluster {
@@ -101,7 +106,6 @@ class StatIdx {
   const std::vector<Station>& getStations() const { return _stations; }
   const std::vector<Group>& getGroups() const { return _groups; }
 
-
  private:
   void addStation(int64_t osmid, const util::geo::DLine& geom,
                   const util::geo::DLine& latLngs, size_t origGroup,
@@ -112,9 +116,12 @@ class StatIdx {
   void initGroups();
   void initSuggestions();
 
+  int nameAttrRel(const std::string& attr) const;
+
   util::geo::DLine getGroupArrow(size_t stat, const util::geo::DPoint& p) const;
 
-  util::geo::Polygon<double> hull(const util::geo::MultiPoint<double>& imp, double rad) const;
+  util::geo::Polygon<double> hull(const util::geo::MultiPoint<double>& imp,
+                                  double rad) const;
 
   std::vector<Station> _stations;
   std::vector<Group> _groups;

src/osmfixer/server/StatServer.cpp

@@ -2,9 +2,9 @@
 // Chair of Algorithms and Data Structures.
 // Authors: Patrick Brosi <brosi@informatik.uni-freiburg.de>
 
-#include <vector>
 #include <codecvt>
 #include <locale>
+#include <vector>
 #include "osmfixer/build.h"
 #include "osmfixer/index.h"
 #include "osmfixer/server/StatServer.h"
@@ -36,6 +36,7 @@ util::http::Answer StatServer::handle(const util::http::Req& req,
           "200 OK", std::string(build_js, build_js + sizeof build_js /
                                                          sizeof build_js[0]));
       a.params["Content-Type"] = "application/javascript; charset=utf-8";
+      a.params["Cache-Control"] = "public, max-age=10000";
     } else if (cmd == "/map") {
       a = handleMapReq(params);
     } else if (cmd == "/heatmap") {
@@ -211,10 +212,9 @@ util::http::Answer StatServer::handleMapReq(const Params& pars) const {
 
     const auto& gret = idx.getGroups(bbox);
     for (const auto& group : gret) {
-      if (group->polyStations.size() == 1 && group->osmid < 2) continue;
       json << sep;
       sep = ',';
-      printGroup(group, did, z < 15, &json);
+      printGroup(group, did, z < 15, false, &json);
     }
   }
 
@@ -295,7 +295,7 @@ void StatServer::printSugg(const Suggestion* sugg, size_t did,
 
 // _____________________________________________________________________________
 void StatServer::printGroup(const Group* group, size_t did, bool simple,
-                            std::ostream* out) const {
+                            bool aggr, std::ostream* out) const {
   const auto& range = _idxs[did].first;
 
   (*out) << "{\"i\":" << group->id + range.gidStart << ",\"g\":[";
@@ -310,11 +310,19 @@ void StatServer::printGroup(const Group* group, size_t did, bool simple,
     }
   }
   (*out) << "]";
-  if (group->osmid == 1) (*out) << ",\"n\":1";
+  bool sugg = group->osmid == 1 || group->suggestions.size() > 0;
+  int err = group->attrErrs.size();
+
+  if (aggr) {
+    for (const auto sid : group->polyStations) {
+      sugg = sugg || _idxs[did].second.getStation(sid)->suggestions.size();
+      err += _idxs[did].second.getStation(sid)->attrErrs.size();
+    }
+  }
 
-  if (group->suggestions.size() > 0) (*out) << ",\"s\":1";
+  if (sugg) (*out) << ",\"s\":1";
+  if (err) (*out) << ",\"e\":" << err;
 
-  if (group->attrErrs.size()) (*out) << ",\"e\":" << group->attrErrs.size();
   (*out) << "}";
 }
 
@@ -375,8 +383,7 @@ size_t StatServer::getDidByGid(size_t gid) const {
 
 // _____________________________________________________________________________
 util::http::Answer StatServer::handleSearch(const Params& pars) const {
-  if (pars.count("q") == 0)
-    throw std::invalid_argument("No query given.");
+  if (pars.count("q") == 0) throw std::invalid_argument("No query given.");
   std::string cb;
   if (pars.count("cb")) cb = pars.find("cb")->second.c_str();
 
@@ -400,15 +407,27 @@ util::http::Answer StatServer::handleSearch(const Params& pars) const {
     const auto& idx = _idxs[did].second;
     size_t idxGid = gid - range.gidStart;
     const auto group = idx.getGroup(idxGid);
-    json << sep << "{\"gid\":" << gid << ",\"score\":" << res.second << ", \"name\":";
-    std::string name = group->uniqueNames.front();
+
+    bool asStat = group->polyStations.size() == 1;
+
+    json << sep << "{\"n\":";
+    std::string name = group->uniqueNames.front().second;
     json << "\"" << util::jsonStringEscape(name) << "\"";
     std::string via = _searchIdx.getName(res.first.second);
     if (name != via) {
-      json << ",\"via\":" << "\"" << util::jsonStringEscape(via) << "\"";
+      json << ",\"v\":"
+           << "\"" << util::jsonStringEscape(via) << "\"";
     }
 
-    json << ",\"latlng\":{\"lat\":" << group->centroid.getY() << ",\"lng\":" << group->centroid.getX() <<"}";
+    if (asStat) {
+      auto stat = idx.getStation(group->polyStations.front());
+      if (stat->osmid < 0) json << ",\"w\":1";
+      json << ",\"s\":";
+      printStation(stat, did, true, &json);
+    } else {
+      json << ",\"g\":";
+      printGroup(group, did, true, true, &json);
+    }
 
     sep = ",";
     json << "}";

src/osmfixer/server/StatServer.h

@@ -13,10 +13,6 @@
 
 namespace osmfixer {
 
-struct IdRange {
-  size_t sidStart, sidEnd, gidStart, gidEnd, suggIdStart, suggIdEnd;
-};
-
 typedef std::map<std::string, std::string> Params;
 
 class StatServer : public util::http::Handler {
@@ -38,7 +34,7 @@ class StatServer : public util::http::Handler {
   util::http::Answer handleSearch(const Params& pars) const;
 
   void printStation(const Station* stat, size_t did, bool simple, std::ostream* out) const;
-  void printGroup(const Group* stat, size_t did, bool simple, std::ostream* out) const;
+  void printGroup(const Group* stat, size_t did, bool simple, bool aggr, std::ostream* out) const;
   void printSugg(const Suggestion* stat, size_t did, std::ostream* out) const;
 
   size_t getDidBySid(size_t sid) const;

web/codes.txt

@@ -0,0 +1,7 @@
+U+0061
+U+0074
+U+0073
+U+0079
+U+002B
+U+002D
+U+2212