lib/rewritten.rb
require 'redis/namespace'
require 'rewritten/version'
require 'rewritten/helpers'
require 'rack/dummy'
require 'rack/url'
require 'rack/record'
require 'rack/html'
require 'rack/subdomain'
require 'rack/canonical'
require 'rewritten/document'
module Rewritten
include Helpers
extend self
# Accepts:
# 1. A 'hostname:port' String
# 2. A 'hostname:port:db' String (to select the Redis db)
# 3. A 'hostname:port/namespace' String (to set the Redis namespace)
# 4. A Redis URL String 'redis://host:port'
# 5. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
# or `Redis::Namespace`.
def redis=(server)
case server
when String
if server =~ /redis\:\/\//
redis = Redis.connect(url: server, thread_safe: true)
else
server, namespace = server.split('/', 2)
host, port, db = server.split(':')
redis = Redis.new(host: host, port: port,
thread_safe: true, db: db)
end
namespace ||= :rewritten
@redis = Redis::Namespace.new(namespace, redis: redis)
when Redis::Namespace
@redis = server
else
@redis = Redis::Namespace.new(:rewritten, redis: server)
end
end
attr_accessor :translate_partial
# Returns the current Redis connection. If none has been created, will
# create a new one.
def redis
return @redis if @redis
self.redis = Redis.respond_to?(:connect) ? Redis.connect : 'localhost:6379'
redis
end
def redis_id
# support 1.x versions of redis-rb
if redis.respond_to?(:server)
redis.server
elsif redis.respond_to?(:nodes) # distributed
redis.nodes.map(&:id).join(', ')
else
redis.client.id
end
end
# The `before_first_fork` hook will be run in the **parent** process
# only once, before forking to run the first job. Be careful- any
# changes you make will be permanent for the lifespan of the
# worker.
#
# Call with a block to set the hook.
# Call with no arguments to return the hook.
def before_first_fork(&block)
block ? (@before_first_fork = block) : @before_first_fork
end
# Set a proc that will be called in the parent process before the
# worker forks for the first time.
attr_writer :before_first_fork
# The `before_fork` hook will be run in the **parent** process
# before every job, so be careful- any changes you make will be
# permanent for the lifespan of the worker.
#
# Call with a block to set the hook.
# Call with no arguments to return the hook.
def before_fork(&block)
block ? (@before_fork = block) : @before_fork
end
# Set the before_fork proc.
attr_writer :before_fork
# The `after_fork` hook will be run in the child process and is passed
# the current job. Any changes you make, therefore, will only live as
# long as the job currently being processed.
#
# Call with a block to set the hook.
# Call with no arguments to return the hook.
def after_fork(&block)
block ? (@after_fork = block) : @after_fork
end
# Set the after_fork proc.
attr_writer :after_fork
def to_s
"Rewritten Client connected to #{redis_id}"
end
# If 'inline' is true Resque will call #perform method inline
# without queuing it into Redis and without any Resque callbacks.
# The 'inline' is false Resque jobs will be put in queue regularly.
def inline?
@inline
end
alias_method :inline, :inline?
attr_writer :inline
#
# translations
#
def add_translation(line, to)
from, flags = line.split(/\s+/)
flags = flags.scan(/\[(\w+)\]/).first if flags
redis.hset("from:#{from}", :to, to)
redis.hset("from:#{from}", :flags, flags) if flags
redis.sadd(:froms, from)
redis.sadd(:tos, to)
score = redis.zcard("to:#{to}") || 0
redis.zadd("to:#{to}", score, from)
end
def add_translations(to, froms)
froms.each { |from| add_translation(from, to) }
end
def num_translations(to)
Rewritten.redis.zcard("to:#{to}")
end
def remove_translation(from, to)
Rewritten.redis.del("from:#{from}")
Rewritten.redis.srem(:froms, from)
Rewritten.redis.zrem("to:#{to}", from)
Rewritten.redis.srem(:tos, to) if num_translations(to) == 0
end
def remove_all_translations(to)
get_all_translations(to).each do |from|
Rewritten.remove_translation(from, to)
end
end
def clear_translations
Rewritten.redis.del(*Rewritten.redis.keys) unless Rewritten.redis.keys.empty?
end
# Returns an array of all known source URLs (that are to translated)
def froms
Array(redis.smembers(:froms))
end
def all_froms
Array(redis.smembers(:froms))
end
def all_tos
Array(Rewritten.redis.smembers(:tos))
end
def translate(from)
redis.hget("from:#{from}", :to)
end
def get_all_translations(to)
Rewritten.redis.zrange("to:#{to}", 0, -1)
end
def get_current_translation(path, tail = nil)
uri = URI.parse(path)
# find directly
translation = Rewritten.z_range("to:#{path}", -1)
translation = Rewritten.z_range("to:#{uri.path}", -1) unless translation
if translation.nil?
if translate_partial && path.count('/') > 1
parts = path.split('/')
shorter_path = parts.slice(0, parts.size - 1).join('/')
appendix = parts.last + (tail ? '/' + tail : '')
return get_current_translation(shorter_path, appendix)
else
return path
end
end
complete_path = (tail ? translation + '/' + tail : translation)
translated_uri = URI.parse(complete_path)
uri.path = translated_uri.path
uri.query = [translated_uri.query, uri.query].compact.join('&')
uri.query = nil if uri.query == ''
uri.to_s
end
# infinitive for translations only!
def infinitive(some_from)
some_from ||= ''
conjugated = some_from.chomp('/')
to = translate(conjugated)
to = translate(conjugated.split('?')[0]) unless to
if to.nil? && translate_partial && conjugated.count('/') > 1
parts = conjugated.split('/')
shorter_path = parts.slice(0, parts.size - 1).join('/')
infinitive(shorter_path)
else
conjugated = get_current_translation(to) if to
conjugated = conjugated.split('?')[0] || ''
conjugated.chomp('/')
end
end
def base_from(some_from)
base_from = some_from.split('?')[0].chomp('/')
if translate(some_from)
some_from
elsif translate(base_from)
base_from
elsif translate_partial && base_from.count('/') > 1
parts = base_from.split('/')
base_from(parts.slice(0, parts.size - 1).join('/'))
end
end
def appendix(some_from)
base = base_from(some_from) || ''
result = some_from.partition(base).last
return '' if result == ''
result.split('?').first.chomp('/')
end
def get_flag_string(from)
Rewritten.redis.hget("from:#{from}", :flags) || ''
end
def flag?(from, c)
return false unless Rewritten.redis.exists("from:#{from}")
get_flag_string(from).index(c) != nil
end
def full_line(from)
flags = get_flag_string(from)
if flags == ''
from
else
"#{from} [#{flags}]"
end
end
def exist_translation_for?(path)
get_current_translation(path) != path
end
def add_hit(path, code, content_type)
h = { path: path, code: code, content_type: content_type }
Rewritten.redis.sadd('hits', encode(h))
end
def all_hits
Rewritten.redis.smembers('hits').map { |e| decode(e) }
end
def includes?(path)
result = Rewritten.redis.hget("from:#{path.chomp('/')}", :to)
result = Rewritten.redis.hget("from:#{path.split('?')[0]}", :to) unless result
if result.nil? && translate_partial && path.count('/') > 1
parts = path.split('/')
includes?(parts.slice(0, parts.size - 1).join('/'))
else
result
end
end
# return the number of froms
def num_froms
redis.scard(:froms).to_i
end
# Does the dirty work of fetching a range of items from a Redis list
# and converting them into Ruby objects.
def z_range(key, start = 0, count = 1)
if count == 1
redis.zrange(key, start, start)[0]
else
Array(redis.zrange(key, start, start + count - 1)).map do |item|
item
end
end
end
# Returns an array of all known Resque queues as strings.
def queues
Array(redis.smembers(:queues))
end
# Returns an array of all known URL targets.
def targets
Array(redis.smembers(:targets))
end
# Given a queue name, completely deletes the queue.
def remove_queue(queue)
redis.srem(:queues, queue.to_s)
redis.del("queue:#{queue}")
end
# Used internally to keep track of which queues we've created.
# Don't call this directly.
def watch_queue(queue)
redis.sadd(:queues, queue.to_s)
end
#
# stats
#
# Returns a hash, similar to redis-rb's #info, of interesting stats.
def info
{
pending: queues.inject(0) { |m, k| m + size(k) },
#:processed => Stat[:processed],
#:queues => queues.size,
#:workers => workers.size.to_i,
#:working => working.size,
#:failed => Stat[:failed],
servers: [redis_id],
environment: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
}
end
# Returns an array of all known Resque keys in Redis. Redis' KEYS operation
# is O(N) for the keyspace, so be careful - this can be slow for big databases.
def keys
redis.keys('*').map do |key|
key.sub("#{redis.namespace}:", '')
end
end
def per_page
20
end
end