app/models/import.rb
# frozen_string_literal: true
# Redmine - project management software
# Copyright (C) 2006- Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
require 'csv'
class Import < ApplicationRecord
has_many :items, :class_name => 'ImportItem', :dependent => :delete_all
belongs_to :user
serialize :settings
before_destroy :remove_file
validates_presence_of :filename, :user_id
validates_length_of :filename, :maximum => 255
DATE_FORMATS = [
'%Y-%m-%d',
'%d/%m/%Y',
'%m/%d/%Y',
'%Y/%m/%d',
'%d.%m.%Y',
'%d-%m-%Y'
]
AUTO_MAPPABLE_FIELDS = {}
def self.menu_item
nil
end
def self.layout
'base'
end
def self.authorized?(user)
user.admin?
end
def initialize(*args)
super
self.settings ||= {}
end
def file=(arg)
return unless arg.present? && arg.size > 0
self.filename = generate_filename
Redmine::Utils.save_upload(arg, filepath)
end
def set_default_settings(options={})
separator = lu(user, :general_csv_separator)
wrapper = '"'
encoding = lu(user, :general_csv_encoding)
if file_exists?
begin
content = File.read(filepath, 256)
separator = [',', ';'].max_by {|sep| content.count(sep)}
wrapper = ['"', "'"].max_by {|quote_char| content.count(quote_char)}
guessed_encoding = Redmine::CodesetUtil.guess_encoding(content)
encoding =
(guessed_encoding && (
Setting::ENCODINGS.detect {|e| e.casecmp?(guessed_encoding)} ||
Setting::ENCODINGS.detect {|e| Encoding.find(e) == Encoding.find(guessed_encoding)}
)) || lu(user, :general_csv_encoding)
rescue => e
end
end
date_format = lu(user, "date.formats.default", :default => "foo")
date_format = DATE_FORMATS.first unless DATE_FORMATS.include?(date_format)
self.settings.merge!(
'separator' => separator,
'wrapper' => wrapper,
'encoding' => encoding,
'date_format' => date_format,
'notifications' => '0'
)
if options.key?(:project_id) && options[:project_id].present?
# Do not fail if project doesn't exist
begin
project = Project.find(options[:project_id])
self.settings.merge!('mapping' => {'project_id' => project.id})
rescue; end
end
end
def to_param
filename
end
# Returns the full path of the file to import
# It is stored in tmp/imports with a random hex as filename
def filepath
if filename.present? && /\A[0-9a-f]+\z/.match?(filename)
File.join(Rails.root, "tmp", "imports", filename)
else
nil
end
end
# Returns true if the file to import exists
def file_exists?
filepath.present? && File.exist?(filepath)
end
# Returns the headers as an array that
# can be used for select options
def columns_options(default=nil)
i = -1
headers.map {|h| [h, i+=1]}
end
# Parses the file to import and updates the total number of items
def parse_file
count = 0
read_items {|row, i| count=i}
update_attribute :total_items, count
count
end
# Reads the items to import and yields the given block for each item
def read_items
i = 0
headers = true
read_rows do |row|
if i == 0 && headers
headers = false
next
end
i+= 1
yield row, i if block_given?
end
end
# Returns the count first rows of the file (including headers)
def first_rows(count=4)
rows = []
read_rows do |row|
rows << row
break if rows.size >= count
end
rows
end
# Returns an array of headers
def headers
first_rows(1).first || []
end
# Returns the mapping options
def mapping
settings['mapping'] || {}
end
# Adds a callback that will be called after the item at given position is imported
def add_callback(position, name, *args)
settings['callbacks'] ||= {}
settings['callbacks'][position] ||= []
settings['callbacks'][position] << [name, args]
save!
end
# Executes the callbacks for the given object
def do_callbacks(position, object)
if callbacks = (settings['callbacks'] || {}).delete(position)
callbacks.each do |name, args|
send :"#{name}_callback", object, *args
end
save!
end
end
# Imports items and returns the position of the last processed item
def run(options={})
max_items = options[:max_items]
max_time = options[:max_time]
current = 0
imported = 0
resume_after = items.maximum(:position) || 0
interrupted = false
started_on = Time.now
read_items do |row, position|
if (max_items && imported >= max_items) || (max_time && Time.now >= started_on + max_time)
interrupted = true
break
end
if position > resume_after
item = items.build
item.position = position
item.unique_id = row_value(row, 'unique_id') if use_unique_id?
if object = build_object(row, item)
if object.save
item.obj_id = object.id
else
item.message = object.errors.full_messages.join("\n")
end
end
item.save!
imported += 1
extend_object(row, item, object) if object.persisted?
do_callbacks(use_unique_id? ? item.unique_id : item.position, object)
end
current = position
end
if imported == 0 || interrupted == false
if total_items.nil?
update_attribute :total_items, current
end
update_attribute :finished, true
remove_file
end
current
end
def unsaved_items
items.where(:obj_id => nil)
end
def saved_items
items.where("obj_id IS NOT NULL")
end
private
def read_rows
return unless file_exists?
csv_options = {:headers => false}
csv_options[:encoding] = settings['encoding'].to_s.presence || 'UTF-8'
csv_options[:encoding] = 'bom|UTF-8' if csv_options[:encoding] == 'UTF-8'
separator = settings['separator'].to_s
csv_options[:col_sep] = separator if separator.size == 1
wrapper = settings['wrapper'].to_s
csv_options[:quote_char] = wrapper if wrapper.size == 1
CSV.foreach(filepath, **csv_options) do |row|
yield row if block_given?
end
end
def row_value(row, key)
if index = mapping[key].presence
row[index.to_i].presence
end
end
def row_date(row, key)
if s = row_value(row, key)
format = settings['date_format']
format = DATE_FORMATS.first unless DATE_FORMATS.include?(format)
Date.strptime(s, format) rescue s
end
end
# Builds a record for the given row and returns it
# To be implemented by subclasses
def build_object(row, item)
end
# Extends object with properties, that may only be handled after it's been
# persisted.
def extend_object(row, item, object)
end
# Generates a filename used to store the import file
def generate_filename
Redmine::Utils.random_hex(16)
end
# Deletes the import file
def remove_file
if file_exists?
begin
File.delete filepath
rescue => e
logger.error "Unable to delete file #{filepath}: #{e.message}" if logger
end
end
end
# Returns true if value is a string that represents a true value
def yes?(value)
value == lu(user, :general_text_yes) || value == '1'
end
def use_unique_id?
mapping['unique_id'].present?
end
end