npolar/api.npolar.no

require "typhoeus"
require "nokogiri"
require "time"
require "json"
require "fileutils"
require "logger"
require "digest"

module Followit

  class TrackerService

    attr_accessor :auth

    include Soap

    URI = "http://total.followit.se/DataAccess/TrackerService.asmx"

    # Download tracks for a user to disk (organised by year and month)
    #
    # Login
    # Get (tracker ids)
    # For each tracker id
    #   GetTrafficDates
    #     Reduce to unique months
    #       GetUnitReportPositions
    #         Save if valid
    #
    def download(destination, earliest=nil, latest=nil)
      if auth.nil?
        raise ArgumentError, "Please set auth -> Followit::AuthService"
      end
      log.debug "Starting #{auth.username} download of Followit XML to #{destination} [#{ earliest.nil? ? '': "earliest date: #{earliest.to_date}"}]"
      auth.login

      tracker_ids = get_tracker_ids
      log.debug "Trackers for #{auth.username} (#{tracker_ids.size}): #{tracker_ids.to_json}"

      tracker_ids.each_with_index {|id, idx|

        if idx > 0 and not auth.nil?
          # login again (to avoid session to expires)
          auth.login
        end
        # log.debug "Tracker: #{id}"

        get_traffic_dates(id, earliest).map {|dt|
          d = dt.to_date
          DateTime.new(d.year, d.month)
          }.uniq.each {| dt |
          begin

            year = dt.year
            date_from = Date.new(dt.year, dt.month)
            date_to = date_from >> 1
            month_text = date_from.month.to_s.gsub(/-01$/, "").rjust(2, "0")

            folder = "#{destination}/#{year}/#{year}-#{month_text}"
            FileUtils.mkdir_p(folder)
            filename = "#{folder}/followit-#{year}-#{month_text}-tracker-#{id}.xml"

            from = dt.to_time.utc.iso8601
            to = DateTime.parse(date_to.to_s).to_time.utc.iso8601

            positions = get_unit_report_positions(id, from, to)

            new_sha1 = Digest::SHA1.hexdigest(positions.to_xml)

            if valid_positions? positions, filename
              if File.exists? filename
                existing_sha1 = Digest::SHA1.file(filename).hexdigest

                if new_sha1 != existing_sha1

                  # Often the difference is just TrafficGPSExtendedId
                  # $ diff followit.se/2014/2014-03/followit-2014-03-tracker-10458.xml followit.se_/2014/2014-03/followit-2014-03-tracker-10458.xml
                  # 1454c1454
                  # <           <TrafficGPSExtendedId>5553977</TrafficGPSExtendedId>
                  # ---
                  # >           <TrafficGPSExtendedId>5551407</TrafficGPSExtendedId>
                  # 1480c1480
                  # <           <TrafficGPSExtendedId>5551408</TrafficGPSExtendedId>
                  # ---
                  # >           <TrafficGPSExtendedId>5553978</TrafficGPSExtendedId>
                  # 1506c1506
                  # <           <TrafficGPSExtendedId>5553979</TrafficGPSExtendedId>
                  # ---
                  # >           <TrafficGPSExtendedId>5551409</TrafficGPSExtendedId>
                  # 1532c1532
                  # <           <TrafficGPSExtendedId>5551410</TrafficGPSExtendedId>
                  # ---
                  # >           <TrafficGPSExtendedId>5553980</TrafficGPSExtendedId>
                  # 1558c1558
                  # <           <TrafficGPSExtendedId>5553981</TrafficGPSExtendedId>
                  # ---
                  # >           <TrafficGPSExtendedId>5551411</TrafficGPSExtendedId>
                  # 1584c1584
                  # <           <TrafficGPSExtendedId>5551412</TrafficGPSExtendedId>
                  # ---
                  # >           <TrafficGPSExtendedId>5553982</TrafficGPSExtendedId>



                    File.open(filename, "wb") { |file| file.write(positions.to_xml)}
                    log.warn "Updated: #{filename}"




                else
                  log.debug "Keeping existing #{filename}"
                end

              else
                File.open(filename, "wb") { |file| file.write(positions.to_xml)}
                log.debug "Validated and saved new data: #{filename}"
              end

            else
              log.error "Invalid unit positions: #{positions}"
            end
          rescue => e
            log.fatal "Failed downloading followit.se #{year}-#{month_text} data for tracker #{id}: #{e}"
          end
        }
      }
      log.debug "Finished #{auth.username} Followit XML download"

    end

    def get_tracker_ids
      get_trackers("//followit:GetResult/followit:Tracker/followit:TrackerId").map {|t|
        t.content
      }.sort
    end

    def get_trackers(xpath="//followit:GetResult/followit:Tracker")
      extract(get, xpath).sort
    end

    def get_trackers_array
      document = Nokogiri::XML(get.body)
      template = Nokogiri::XSLT(File.read(File.join(__dir__, "..", "..", "..", "xslt", "followit-trackers.xslt")))
      JSON.parse(template.apply_to(document)) # https://github.com/sparklemotion/nokogiri/issues/247
    end

    def get
      execute(request(get_tracker_ids_envelope))
    end

    def get_traffic_dates(tracker_id, earliest=nil)
      traffic_dates = extract(execute(request(get_traffic_dates_envelope(tracker_id))), "//followit:GetTrafficDatesResult/followit:dateTime").map {|dt|
        DateTime.parse(dt)
      }.sort.reverse
      if earliest.nil?
        traffic_dates
      else
        traffic_dates.reject {|dt| dt < earliest }
      end

    end

    def traffic_dates(tracker_id, &block)
      response = execute(request(get_traffic_dates_envelope(tracker_id)))
      if block_given?
        block.call(response.body)
      else
        response
      end
    end

    def get_traffic_dates_for_all_trackers(ids=nil, &block)
      dates = {}
      ids = ids.nil? ? get_tracker_ids : ids

      ids.each {|id|
        dates[id] = get_traffic_dates(id)
      }
      dates
    end

    def get_unit_report_positions(tracker_id, from="1900-01-01T00:00:00Z", to="2100-01-01T00:00:00Z")
      extract(execute(request(get_unit_report_positions_envelope(tracker_id, from, to))), "/soap:Envelope")
    end

    def positions_from_xml(xml)
      if File.exists? xml
        log.debug "Creating JSON from XML file #{xml}"
        xml = File.read(xml)
      end
      document = Nokogiri::XML(xml)
      template = Nokogiri::XSLT(File.read(File.join(__dir__, "..", "..", "..", "xslt", "followit-json.xslt")))
      JSON.parse(template.apply_to(document)) # https://github.com/sparklemotion/nokogiri/issues/247
    end

    def valid_positions? (positions, f=nil)
      @count = xpath("count(//followit:UnitReportPosition)").to_i
      log.debug "#{@count} positions [#{File.basename(f)}]"
      @count > 0
    end

    protected

    #<Get xmlns="http://tempuri.org/">
    #  <queryData>
    #    <Filter>%%</Filter>
    #    <Top>1000</Top>
    #  </queryData>
    #</Get>
    def get_tracker_ids_envelope(filter="%%", top=1000)
      envelope do |xml|
        xml.Get(xmlns: NAMESPACE) do
          xml.queryData do
            xml.Filter filter
            xml.Top top
          end
        end
      end
    end

    def get_unit_report_positions_envelope(tracker_id, from, to)
      envelope do |xml|
        xml.GetUnitReportPositions(xmlns: NAMESPACE) do
          xml.trackerId tracker_id
          xml.fromDate from
          xml.toDate to
        end
      end
    end

    def get_traffic_dates_envelope(tracker_id)
      envelope do |xml|
        xml.GetTrafficDates(xmlns: NAMESPACE) do
          xml.trackerId tracker_id
        end
      end
    end

  end
end