app/models/active_road/osm_pbf_importer_level_db.rb
require 'leveldb-native'
require 'csv'
module ActiveRoad
class OsmPbfImporterLevelDb
include OsmPbfImporter
@@csv_batch_size = 100000
cattr_reader :csv_batch_size
attr_reader :ways_database_path, :nodes_database_path, :physical_roads_database_path, :junctions_database_path, :pbf_file, :split_ways
def initialize(pbf_file, split_ways = false, prefix_path = "/tmp/")
@pbf_file = pbf_file
@split_ways = split_ways
FileUtils.mkdir_p(prefix_path) if !Dir.exists?(prefix_path)
@nodes_database_path = prefix_path + "osm_pbf_nodes_leveldb"
@ways_database_path = prefix_path + "osm_pbf_ways_leveldb"
@junctions_database_path = prefix_path + "osm_pbf_junctions_leveldb"
@physical_roads_database_path = prefix_path + "osm_pbf_physical_roads_leveldb"
end
def nodes_database
@nodes_database ||= LevelDBNative::DB.make nodes_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end
def close_nodes_database
nodes_database.close!
end
def delete_nodes_database
FileUtils.remove_entry nodes_database_path if File.exists?(nodes_database_path)
end
def ways_database
@ways_database ||= LevelDBNative::DB.make ways_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end
def close_ways_database
ways_database.close!
end
def delete_ways_database
FileUtils.remove_entry ways_database_path if File.exists?(ways_database_path)
end
def junctions_database
@junctions_database ||= LevelDBNative::DB.make junctions_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end
def close_junctions_database
junctions_database.close!
end
def delete_junctions_database
FileUtils.remove_entry junctions_database_path if File.exists?(junctions_database_path)
end
def physical_roads_database
@physical_roads_database ||= LevelDBNative::DB.make physical_roads_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end
def close_physical_roads_database
physical_roads_database.close!
end
def delete_physical_roads_database
FileUtils.remove_entry physical_roads_database_path if File.exists?(physical_roads_database_path)
end
def display_time(time_difference)
Time.at(time_difference.to_i).utc.strftime "%H:%M:%S"
end
def import
delete_nodes_database
delete_ways_database
delete_junctions_database
delete_physical_roads_database
leveldb_import
postgres_import
close_nodes_database
close_ways_database
close_junctions_database
close_physical_roads_database
end
def leveldb_import
# Save nodes in temporary file
backup_nodes
# Update nodes with ways in temporary file
update_nodes_with_way
# Save ways in temporary file
backup_ways
end
def postgres_import
# Save nodes in junctions
iterate_nodes
# Save relations in boundary
backup_relations_pgsql if split_ways
# Save ways in physical roads
iterate_ways
save_junctions_and_physical_roads_temporary
save_physical_road_conditionnal_costs_and_junctions
# Split and affect boundary to each way
split_way_with_boundaries if split_ways
# Save logical roads from physical roads
backup_logical_roads_pgsql if split_ways
end
def backup_nodes
Rails.logger.info "Begin to backup nodes in LevelDB nodes_database in #{nodes_database_path}"
start = Time.now
nodes_parser = ::PbfParser.new(pbf_file)
nodes_counter = 0
nodes_hash = {}
# Process the file until it finds any node
nodes_parser.next until nodes_parser.nodes.any?
until nodes_parser.nodes.empty?
nodes_database.batch do |batch|
last_node = nodes_parser.nodes.last
nodes_parser.nodes.each do |node|
nodes_counter+= 1
select_tags = selected_tags(node[:tags], @@nodes_selected_tags_keys)
batch[ node[:id].to_s ] = Marshal.dump(Node.new(node[:id].to_s, node[:lon], node[:lat], select_tags["addr:housenumber"], [], false, select_tags))
end
end
# When there's no more fileblocks to parse, #next returns false
# This avoids an infinit loop when the last fileblock still contains ways
break unless nodes_parser.next
end
Rails.logger.info "Finish to backup #{nodes_counter} nodes in LevelDB nodes_database in #{display_time(Time.now - start)} seconds"
end
def update_nodes_with_way
Rails.logger.info "Update way in nodes in LevelDB"
start = Time.now
ways_parser = ::PbfParser.new(pbf_file)
ways_counter = 0
# Process the file until it finds any way.
ways_parser.next until ways_parser.ways.any?
# Once it found at least one way, iterate to find the remaining ways.
until ways_parser.ways.empty?
nodes_readed = {}
nodes_database.batch do |batch|
ways_parser.ways.each do |way|
way_id = way[:id].to_s
if way.key?(:tags) && required_way?(@@way_for_physical_road_required_tags_keys, way[:tags])
# Don't add way to nodes if a way is a boundary
select_tags = selected_tags(way[:tags], @@way_selected_tags_keys)
node_ids = way.key?(:refs) ? way[:refs].collect(&:to_s) : []
if node_ids.present? && node_ids.size > 1
ways_counter+= 1
node_ids.each do |node_id|
if nodes_readed.has_key?(node_id)
node = nodes_readed[node_id]
else
node = Marshal.load(nodes_database[node_id])
end
node.add_way(way_id)
node.end_of_way = true if [node_ids.first, node_ids.last].include?(node_id)
nodes_readed[node_id] = node
end
end
end
end
nodes_readed.each_pair do |node_readed_id, node_readed|
batch[node_readed_id] = Marshal.dump(node_readed)
end
end
# When there's no more fileblocks to parse, #next returns false
# This avoids an infinit loop when the last fileblock still contains ways
break unless ways_parser.next
end
Rails.logger.info "Finish to update #{ways_counter} ways in nodes in LevelDB in #{display_time(Time.now - start)} seconds"
end
# def update_node_with_way(way_id, node_ids)
# # Update node data with way id
# node_ids.each do |node_id|
# node = Marshal.load(nodes_database[node_id])
# node.add_way(way_id)
# node.end_of_way = true if [node_ids.first, node_ids.last].include?(node_id)
# nodes_database[node_id] = Marshal.dump(node)
# end
# end
def backup_ways
Rails.logger.info "Begin to backup ways in LevelDB"
start = Time.now
ways_parser = ::PbfParser.new(pbf_file)
ways_counter = 0
# Process the file until it finds any way.
ways_parser.next until ways_parser.ways.any?
# Once it found at least one way, iterate to find the remaining ways.
until ways_parser.ways.empty?
ways_database.batch do |batch|
ways_parser.ways.each do |way|
way_id = way[:id].to_s
if way.key?(:tags) && required_way?(@@way_required_tags_keys, way[:tags])
select_tags = selected_tags(way[:tags], @@way_selected_tags_keys)
opt_tags = selected_tags(way[:tags], @@way_optionnal_tags_keys)
node_ids = way.key?(:refs) ? way[:refs].collect(&:to_s) : []
way = Way.new( way_id, node_ids, car?(opt_tags), bike?(opt_tags), train?(opt_tags), pedestrian?(opt_tags), select_tags["name"], select_tags["maxspeed"], select_tags["oneway"], select_tags["boundary"], select_tags["admin_level"], select_tags["addr:housenumber"], opt_tags )
ways_splitted = (way.boundary.present? || way.addr_housenumber.present?) ? [way] : split_way_with_nodes(way) # Don't split boundary and adress way
ways_splitted.each do |way_splitted|
ways_counter+= 1
batch[ way_splitted.id ] = Marshal.dump( way_splitted )
end
end
end
end
# When there's no more fileblocks to parse, #next returns false
# This avoids an infinit loop when the last fileblock still contains ways
break unless ways_parser.next
end
Rails.logger.info "Finish to backup #{ways_counter} ways in LevelDB in #{display_time(Time.now - start)} seconds"
end
def split_way_with_nodes(way)
nodes_used = []
nodes = []
# Get nodes really used and all nodes (used and for geometry need) for a way
way.nodes.each_with_index do |node_id, index|
node = Marshal.load( nodes_database[node_id.to_s] )
nodes << node
nodes_used << index if node.used?
end
ways_nodes = []
# Split way between each nodes used
if split_ways
nodes_used.each_with_index do |before_node, index|
ways_nodes << nodes.values_at(before_node..nodes_used[ index + 1]) if before_node != nodes_used.last
end
else
ways_nodes = [nodes]
end
ways_splitted = []
ways_nodes.each_with_index do |way_nodes, index|
way_tags = way.options.dup
way_tags["first_node_id"] = way_nodes.first.id
way_tags["last_node_id"] = way_nodes.last.id
# Don't add way if node_ids contains less than 2 nodes
if way_nodes.present? && way_nodes.size > 1
ways_splitted << Way.new( way.id + "-#{index}", way_nodes.collect(&:id), way.car, way.bike, way.train, way.pedestrian, way.name, way.maxspeed, way.oneway, way.boundary, way.admin_level, way.addr_housenumber, way_tags )
end
end
ways_splitted
end
def iterate_nodes
Rails.logger.debug "Begin to backup nodes in PostgreSql"
start = Time.now
nodes_counter = street_numbers_counter = 0
junctions_values = []
street_numbers_values = []
nodes_database_size = nodes_database.count
# traverse records by iterator
junction_columns = ["objectid", "geometry", "created_at", "updated_at"]
street_number_columns = ["objectid", "geometry", "number", "tags", "created_at", "updated_at"]
CSV.open("/tmp/junctions.csv", "wb:UTF-8") do |junctions_csv|
CSV.open("/tmp/street_numbers.csv", "wb:UTF-8") do |street_numbers_csv|
junctions_csv << junction_columns
street_numbers_csv << street_number_columns
nodes_database.each { |key, value|
node = Marshal.load(value)
geometry = GeoRuby::SimpleFeatures::Point.from_x_y( node.lon, node.lat, 4326) if( node.lon && node.lat )
if node.ways.present? && (node.ways.count >= 2 || node.end_of_way == true ) # Take node with at least two ways or at the end of a way
nodes_counter += 1
junctions_csv << [ node.id, geometry.as_hex_ewkb, Time.now, Time.now ]
end
if node.addr_housenumber.present?
street_numbers_counter += 1
street_numbers_csv << [ node.id, geometry.as_hex_ewkb, node.addr_housenumber, "#{node.tags.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
end
}
end
end
ActiveRoad::Junction.transaction do
ActiveRoad::Junction.pg_copy_from "/tmp/junctions.csv"
end
ActiveRoad::StreetNumber.transaction do
ActiveRoad::StreetNumber.pg_copy_from "/tmp/street_numbers.csv"
end
Rails.logger.info "Finish to backup #{nodes_counter} nodes and #{street_numbers_counter} street numbers in PostgreSql in #{display_time(Time.now - start)} seconds"
end
def iterate_ways
Rails.logger.info "Begin to backup ways in PostgreSql"
start = Time.now
ways_counter = 0
street_numbers_counter = 0
ways_database_size = ways_database.count
# traverse records by iterator
physical_road_columns = ["objectid", "car", "bike", "train", "pedestrian", "name", "geometry", "boundary_id", "tags", "created_at", "updated_at"]
street_number_columns = ["objectid", "geometry", "number", "tags", "created_at", "updated_at"]
CSV.open("/tmp/physical_roads.csv", "wb:UTF-8") do |physical_roads_csv|
CSV.open("/tmp/street_numbers2.csv", "wb:UTF-8") do |street_numbers_csv|
physical_roads_csv << physical_road_columns
street_numbers_csv << street_number_columns
ways_database.each { |key, value|
way = Marshal.load(value)
unless way.boundary.present? # Use ways not used in relation for boundaries
nodes = []
way.nodes.each_with_index do |node_id, index|
node = Marshal.load( nodes_database[node_id.to_s] )
nodes << node
end
way_geometry = way_geometry(nodes)
if way.addr_housenumber.present? # If ways with adress
street_numbers_counter += 1
street_numbers_csv << [ way.id, way_geometry.envelope.center.as_hex_ewkb, way.addr_housenumber, "#{way.options.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
else
ways_counter += 1
way_boundary = way.boundary.present? ? way.boundary.to_i : nil
physical_roads_csv << [ way.id, way.car, way.bike, way.train, way.pedestrian, way.name, way_geometry.as_hex_ewkb, way_boundary, "#{way.options.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
end
end
}
end
end
# Save physical roads
ActiveRoad::PhysicalRoad.transaction do
ActiveRoad::PhysicalRoad.pg_copy_from "/tmp/physical_roads.csv"
end
ActiveRoad::StreetNumber.transaction do
ActiveRoad::StreetNumber.pg_copy_from "/tmp/street_numbers2.csv"
end
Rails.logger.info "Finish to backup #{ways_counter} ways and #{street_numbers_counter} street numbers in PostgreSql in #{display_time(Time.now - start)} seconds"
end
def save_junctions_and_physical_roads_temporary
Rails.logger.info "Begin to backup physical_roads and junctions in LevelDb"
start = Time.now
junctions_database.batch do |batch|
ActiveRoad::Junction.select("id,objectid").find_each do |junction|
junctions_database[junction.objectid] = junction.id.to_s
end
end
physical_roads_database.batch do |batch|
ActiveRoad::PhysicalRoad.select("id,objectid").find_each do |physical_road|
physical_roads_database[physical_road.objectid] = physical_road.id.to_s
end
end
Rails.logger.info "Finish to backup physical_roads and junctions in LevelDb in #{display_time(Time.now - start)} seconds"
end
def save_physical_road_conditionnal_costs_and_junctions
Rails.logger.info "Begin to backup ways in PostgreSql"
start = Time.now
physical_road_conditionnal_costs_counter = junctions_physical_roads_counter = 0
physical_road_conditionnal_cost_columns = ["tags", "cost", "physical_road_id"]
junction_physical_road_columns = ["physical_road_id", "junction_id"]
CSV.open("/tmp/physical_road_conditionnal_costs.csv", "wb:UTF-8") do |physical_road_conditionnal_costs_csv|
CSV.open("/tmp/junctions_physical_roads.csv", "wb:UTF-8") do |junctions_physical_roads_csv|
physical_road_conditionnal_costs_csv << physical_road_conditionnal_cost_columns
junctions_physical_roads_csv << junction_physical_road_columns
ways_database.each { |key, value|
way = Marshal.load(value)
# Save physical road conditionnal cost not for boundaries or street numbers
unless way.boundary.present? || way.addr_housenumber.present?
way_conditionnal_costs = physical_road_conditionnal_costs(way)
way_conditionnal_costs.each do |way_conditionnal_cost|
physical_road_conditionnal_costs_counter += 1
physical_road_conditionnal_costs_csv << way_conditionnal_cost + [ physical_roads_database[way.id] ]
end
way.nodes.each do |node_id|
junction_id = junctions_database[node_id]
junctions_physical_roads_counter += 1
junctions_physical_roads_csv << [ physical_roads_database[way.id], junction_id ] if junction_id.present?
end
end
}
end
end
# Save physical road conditionnal costs
ActiveRoad::PhysicalRoadConditionnalCost.transaction do
ActiveRoad::PhysicalRoadConditionnalCost.pg_copy_from "/tmp/physical_road_conditionnal_costs.csv"
end
# Save physical road and junctions link
ActiveRoad::JunctionsPhysicalRoad.transaction do
ActiveRoad::JunctionsPhysicalRoad.pg_copy_from "/tmp/junctions_physical_roads.csv"
end
Rails.logger.info "Finish to backup #{junctions_physical_roads_counter} junctions_physical_roads and #{physical_road_conditionnal_costs_counter} physical_road_conditionnal_costs in PostgreSql in #{display_time(Time.now - start)} seconds"
end
def split_way_with_boundaries
Rails.logger.info "Begin to split and affect boundaries to ways in PostgreSql"
start = Time.now
# Update physical roads entirely contains in boundaries
ActiveRoad::PhysicalRoad.connection.select_all("SELECT physical_road.id AS physical_road_id, boundary.id AS boundary_id FROM physical_roads physical_road, boundaries boundary WHERE ST_Covers( boundary.geometry, physical_road.geometry)").each_slice(@@pg_batch_size) do |group|
ActiveRoad::PhysicalRoad.transaction do
group.each do |element|
ActiveRoad::PhysicalRoad.update(element["physical_road_id"], :boundary_id => element["boundary_id"])
end
end
end
if split_ways
simple_ways = []
simple_ways_not_line_string = 0
# Fix : Produce 2 ways when way is tangent to boundary borders for each boundary
# Get geometries in boundary
sql = "SELECT b.id AS boundary_id, p.id AS physical_road_id, p.objectid AS physical_road_objectid, p.tags AS physical_road_tags, ST_AsText(p.geometry) AS physical_road_geometry,
j1.objectid AS departure_objectid, ST_AsText(j1.geometry) AS departure_geometry,
j2.objectid AS arrival_objectid, ST_AsText(j2.geometry) AS arrival_geometry,
ST_AsText( (ST_Dump(ST_Intersection( p.geometry , b.geometry))).geom ) AS intersection_geometry
FROM physical_roads p, boundaries b, junctions j1, junctions j2, junctions_physical_roads jp, junctions_physical_roads jp2
WHERE p.boundary_id IS NULL AND ST_Crosses( b.geometry, p.geometry)
AND j1.id = jp.junction_id AND p.id = jp.physical_road_id AND ST_Equals(ST_StartPoint(p.geometry), j1.geometry)
AND j2.id = jp2.junction_id AND p.id = jp2.physical_road_id AND ST_Equals(ST_EndPoint(p.geometry), j2.geometry)".gsub(/^( |\t)+/, "")
ActiveRoad::PhysicalRoad.connection.select_all( sql ).each do |result|
intersection_geometry = GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['intersection_geometry']}")
# Not take in consideration point intersection!!
if intersection_geometry.class == GeoRuby::SimpleFeatures::LineString
simple_way = SimpleWay.new(result["boundary_id"], result["physical_road_id"], result["physical_road_objectid"], result["physical_road_tags"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['physical_road_geometry']}"), result["departure_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['departure_geometry']}"), result["arrival_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['arrival_geometry']}"), intersection_geometry )
# Delete boucle line string Ex : 9938647-4
simple_ways << simple_way if simple_way.departure != simple_way.arrival
else
simple_ways_not_line_string += 1
end
end
# Get geometries not in boundaries
sql = "SELECT ST_AsText( (ST_Dump(difference_geometry)).geom ) AS difference_geometry, v.id AS physical_road_id, v.objectid AS physical_road_objectid, v.tags AS physical_road_tags, ST_AsText(v.geometry) AS physical_road_geometry,
j1.objectid AS departure_objectid, ST_AsText(j1.geometry) AS departure_geometry,
j2.objectid AS arrival_objectid, ST_AsText(j2.geometry) AS arrival_geometry
FROM
( SELECT pr.id, pr.objectid, pr.tags, pr.geometry, pr.boundary_id, ST_Difference( pr.geometry, ST_Union( b.geometry)) as difference_geometry
FROM physical_roads pr, boundaries b
WHERE pr.boundary_id IS NULL AND ST_Crosses( b.geometry, pr.geometry)
GROUP BY pr.id, pr.geometry) v,
junctions j1, junctions j2, junctions_physical_roads jp, junctions_physical_roads jp2
WHERE j1.id = jp.junction_id AND v.id = jp.physical_road_id AND ST_Equals(ST_StartPoint(v.geometry), j1.geometry)
AND j2.id = jp2.junction_id AND v.id = jp2.physical_road_id AND ST_Equals(ST_EndPoint(v.geometry), j2.geometry)
AND NOT ST_IsEmpty(difference_geometry)".gsub(/^( |\t)+/, "")
ActiveRoad::PhysicalRoad.connection.select_all( sql ).each do |result|
difference_geometry = GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['difference_geometry']}")
if difference_geometry.class == GeoRuby::SimpleFeatures::LineString
simple_way = SimpleWay.new(nil, result["physical_road_id"], result["physical_road_objectid"], result["physical_road_tags"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['physical_road_geometry']}"), result["departure_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['departure_geometry']}"), result["arrival_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['arrival_geometry']}"), difference_geometry )
# Delete boucle line string Ex : 9938647-4
simple_ways << simple_way if simple_way.departure != simple_way.arrival
else
simple_ways_not_line_string += 1
end
end
# Prepare reordering ways
simple_ways_by_old_physical_road_id = simple_ways.group_by{|sw| sw.old_physical_road_id}
# Hack : in the code we take the first one which has an intersection point and it deletes
# dual segment tangent on the boundary borders
simple_ways_by_old_physical_road_id.each do |old_physical_road_id, ways|
ways.each do |way|
if way.departure == way.old_departure_geometry
way.departure_objectid = way.old_departure_objectid
way.previous = nil
else
way.departure_objectid = way.default_departure_objectid
way.previous = ways.detect{ |select_way| select_way.arrival == way.departure }
end
if way.arrival == way.old_arrival_geometry
way.arrival_objectid = way.old_arrival_objectid
way.next = nil
else
way.arrival_objectid = way.default_arrival_objectid
way.next = ways.detect{ |select_way| select_way.departure == way.arrival }
end
end
end
# Save new ways and junctions
#physical_roads ||= ActiveRoad::PhysicalRoad.where(:objectid => simple_ways_by_old_physical_road_id.keys).includes(:conditionnal_costs)
simple_ways_by_old_physical_road_id.each_slice(1000) { |group|
ActiveRoad::PhysicalRoad.transaction do
group.each do |old_physical_road_id, ways|
#puts ways.sort.inspect
next_way = ways.detect{ |select_way| select_way.previous == nil }
way_counter = 0
junction_counter = 0
while next_way != nil
start = Time.now
#old_physical_road = physical_roads.where(:id => old_physical_road_id)
#physical_road.conditionnal_costs = old_physical_road.conditionnal_costs
# Create departure
if next_way.previous != nil
departure = ActiveRoad::Junction.where(:objectid => "#{next_way.departure_objectid}-#{junction_counter}").first_or_create( :geometry => next_way.departure )
junction_counter += 1
else
departure = ActiveRoad::Junction.find_by_objectid(next_way.departure_objectid)
end
# Create arrival
if next_way.next != nil
arrival = ActiveRoad::Junction.where(:objectid => "#{next_way.arrival_objectid}-#{junction_counter}").first_or_create( :geometry => next_way.arrival )
else
arrival = ActiveRoad::Junction.find_by_objectid(next_way.arrival_objectid)
end
old_physical_road_tags = next_way.old_physical_road_tags_hash
old_physical_road_tags["first_node_id"] = departure.objectid
old_physical_road_tags["last_node_id"] = arrival.objectid
physical_road = ActiveRoad::PhysicalRoad.create! :objectid => "#{next_way.old_physical_road_objectid}-#{way_counter}", :boundary_id => next_way.boundary_id, :geometry => next_way.geometry, :tags => old_physical_road_tags
# Add departure and arrival to physical road
physical_road.junctions << [departure, arrival]
way_counter += 1
if way_counter > ways.size
Rails.logger.error "Infinite boucle when save physical road splitted with boundaries"
raise Exception.new "Infinite boucle when save physical road splitted with boundaries"
end
next_way = next_way.next
end
end
end
}
# Delete old ways
ActiveRoad::PhysicalRoad.destroy(simple_ways_by_old_physical_road_id.keys)
end
Rails.logger.info "Finish to split and affect boundaries to ways in PostgreSql in #{display_time(Time.now - start)} seconds"
end
class SimpleWay
include Comparable
attr_accessor :boundary_id, :old_physical_road_id, :old_physical_road_objectid, :old_physical_road_tags, :old_physical_road_geometry, :old_departure_objectid, :old_departure_geometry, :old_arrival_objectid, :old_arrival_geometry, :departure_objectid, :arrival_objectid, :geometry, :next, :previous
def initialize(boundary_id, old_physical_road_id, old_physical_road_objectid, old_physical_road_tags, old_physical_road_geometry, old_departure_objectid, old_departure_geometry, old_arrival_objectid, old_arrival_geometry, geometry)
@boundary_id = boundary_id
@old_physical_road_id = old_physical_road_id
@old_physical_road_objectid = old_physical_road_objectid
@old_physical_road_tags = old_physical_road_tags || ""
@old_physical_road_geometry = old_physical_road_geometry
@old_departure_objectid = old_departure_objectid
@old_departure_geometry = old_departure_geometry
@old_arrival_objectid = old_arrival_objectid
@old_arrival_geometry = old_arrival_geometry
@geometry = geometry
end
def old_physical_road_tags_hash
#Fix tags build from string
tags = {}.tap do |tags|
old_physical_road_tags.split(',').each do |pair|
key, value = pair.split("=>")
tags[key.gsub(/\W/, "")] = value.gsub(/\W/, "")
end
end
end
def departure
#puts "geometry class #{geometry.class}, value #{geometry.inspect}"
geometry.points.first if geometry
end
def arrival
geometry.points.last if geometry
end
def default_departure_objectid
"#{old_departure_objectid}-#{old_arrival_objectid}"
end
def default_arrival_objectid
"#{old_departure_objectid}-#{old_arrival_objectid}"
end
def <=>(another)
# puts "self : #{self.departure.inspect}, #{self.arrival.inspect}"
# puts "another : #{another.departure.inspect}, #{another.arrival.inspect}"
# puts old_physical_road_geometry.points.inspect
# puts old_physical_road_geometry.points.index(another.arrival).inspect
# puts old_physical_road_geometry.points.index(self.departure).inspect
if self.departure == another.arrival || old_physical_road_geometry.points.index(another.arrival) < old_physical_road_geometry.points.index(self.departure)
1
elsif self.arrival == another.departure || old_physical_road_geometry.points.index(self.arrival) < old_physical_road_geometry.points.index(another.departure)
-1
else
nil
end
end
end
def way_geometry(nodes)
points = []
nodes.each do |node|
points << GeoRuby::SimpleFeatures::Point.from_x_y(node.lon, node.lat, 4326)
end
GeoRuby::SimpleFeatures::LineString.from_points(points, 4326) if points.present? && 1 < points.count
end
def find_boundary(way_geometry)
ActiveRoad::Boundary.first_contains(way_geometry)
end
def backup_relations_pgsql
Rails.logger.info "Begin to backup relations in PostgreSql"
start = Time.now
relations_parser = ::PbfParser.new(pbf_file)
boundaries_counter = 0
# traverse records by iterator
boundary_columns = ["objectid", "geometry", "name", "admin_level", "postal_code", "insee_code"]
# Process the file until it finds any relation.
relations_parser.next until relations_parser.relations.any?
# Once it found at least one relation, iterate to find the remaining relations.
CSV.open("/tmp/boundaries.csv", "wb:UTF-8") do |boundary_csv|
boundary_csv << boundary_columns
until relations_parser.relations.empty?
relations_parser.relations.each do |relation|
if relation.key?(:tags) && required_relation?(relation[:tags])
tags = selected_tags(relation[:tags], @@relation_selected_tags_keys)
# Use tags["admin_level"] == "8" because catholic boundaries exist!!
if tags["admin_level"] == "8" && tags["boundary"] == "administrative"
boundaries_counter += 1
outer_ways = {}
inner_ways = {}
begin
relation[:members][:ways].each do |member_way|
way_data = ways_database[ member_way[:id].to_s ]
way = nil
nodes = []
if way_data.present?
way = Marshal.load(way_data)
way.nodes.each do |node_id|
node = Marshal.load( nodes_database[node_id.to_s] )
nodes << node
end
else
raise StandardError, "Geometry error : impossible to find way #{member_way[:id]} for relation #{tags["name"]} with id #{relation[:id]}"
end
if member_way[:role] == "inner"
inner_ways[ member_way[:id] ] = way_geometry(nodes)
elsif member_way[:role] == "outer"
outer_ways[ member_way[:id] ] = way_geometry(nodes)
else # Fix : lot of boundaries have no tags role
outer_ways[ member_way[:id] ] = way_geometry(nodes)
end
end
boundary_polygons = extract_relation_polygon(outer_ways.values, inner_ways.values)
if boundary_polygons.present?
boundary_geometry = GeoRuby::SimpleFeatures::MultiPolygon.from_polygons( boundary_polygons ).as_hex_ewkb
boundary_csv << [ relation[:id], boundary_geometry, tags["name"], tags["admin_level"], tags["addr:postcode"], tags["ref:INSEE"] ]
end
rescue StandardError => e
Rails.logger.error "Geometry error : impossible to build polygon for relation #{tags["name"]} with id #{relation[:id]} : #{e.message}"
end
end
end
end
# When there's no more fileblocks to parse, #next returns false
# This avoids an infinit loop when the last fileblock still contains relations
break unless relations_parser.next
end
end
ActiveRoad::Boundary.transaction do
ActiveRoad::Boundary.pg_copy_from "/tmp/boundaries.csv"
end
Rails.logger.info "Finish to backup #{boundaries_counter} boundaries in PostgreSql in #{display_time(Time.now - start)} seconds"
end
def backup_logical_roads_pgsql
Rails.logger.info "Begin to backup logical roads in PostgreSql"
start = Time.now
logical_roads_counter = 0
saved_name = nil
saved_boundary = nil
saved_logical_road = nil
ActiveRoad::PhysicalRoad.where("physical_roads.name IS NOT NULL OR physical_roads.boundary_id IS NOT NULL").select("name,boundary_id,id").order(:boundary_id,:name).find_in_batches(batch_size: 2000) do |group|
ActiveRoad::LogicalRoad.transaction do
group.each do |physical_road|
not_same_name = (saved_name != physical_road.name)
not_same_boundary = (saved_boundary != physical_road.boundary_id)
saved_name = physical_road.name if not_same_name
saved_boundary = physical_road.boundary_id if not_same_boundary
if not_same_name || not_same_boundary
logical_roads_counter += 1
saved_logical_road = ActiveRoad::LogicalRoad.create(:name => saved_name, :boundary_id => saved_boundary)
end
physical_road.update_column(:logical_road_id, saved_logical_road.id) if saved_logical_road.present?
end
end
end
Rails.logger.info "Finish to backup #{logical_roads_counter} logical roads in PostgreSql in #{ display_time(Time.now - start)} seconds"
end
end
end