src/types/index.js
// @flow
import type { DeleteModeType } from "./DeleteModeTypes";
import type { ClassifyType } from "./ClassifyTypes";
import type { FileState } from "./FileStates";
import type { ReasonType } from "./ReasonTypes";
import type {
FacePPGender,
FacePPGlass,
FacePPResult,
NsfwJsResult,
GenderClass,
AgeClass,
DeepLearningMode,
DeepLearningLogicalOperation,
NsfwType,
FaceApiModelName
} from "./DeepLearningTypes";
import type { ProcessState } from "./ProcessStates";
export type Exact<T> = T & $Shape<T>;
// TODO: flow-typed
export type Logger = {
level: string,
trace: () => void,
debug: () => void,
info: () => void,
warn: () => void,
fatal: () => void
};
export type HashRow = {
hash: string,
p_hash: ?string,
d_hash: ?string,
p_hash_distance?: number | false,
d_hash_distance?: number | false,
width: number,
height: number,
ratio: number,
timestamp: number,
name: string,
to_path: string,
from_path: string,
size: number,
state: number,
process_state: ?string
};
export type ProcessStateRow = {
hash: string,
meta: string,
/**
* 0: file exists or not exists
* -1: file exists
* -2: file exists and cloud uploaded
* -3: file not exists and cloud uploaded
* 1: file missing
* 2: file missing
*/
missing: number,
orientation: number,
trim: string,
view_date: number,
view_count: number,
rating: number,
score: ProcessState,
feature: ProcessState,
detect: ProcessState,
nsfwjs: ProcessState,
facepp: ProcessState,
facepp_face_count: number,
acd_id: string,
acd_md5: string
};
export type ProcessStates = {
missing: boolean,
facePP: boolean,
nsfwJs: boolean
};
export type FileInfo = {
facePP?: {
result: FacePPResult,
version: number
},
nsfwJs?: {
results: NsfwJsResult[],
version: number
},
version?: number,
hash: string,
p_hash: ?string,
d_hash: ?string,
damaged: boolean,
width: number,
height: number,
ratio: number,
size: number,
timestamp: number,
name: string,
type: ClassifyType,
to_path: string,
from_path: string,
state: FileState,
process_state: ?string
};
export type IsAcceptableFunction = FileInfo => boolean;
/** Deep learning related configuration */
export type DeepLearningConfig = {
/** resize image before face++ request */
facePPResizedImageSize: number,
facePPDbTableName: string,
/** face++ judge function. Return true if accepted. */
facePPJudgeFunction: FacePPResult => boolean,
/** face++ api domain */
facePPDomain: string,
facePPDetectApiPath: string,
facePPApiKey: string,
facePPApiSecret: string,
facePPFaceAttributes: [
"gender",
"age",
"smiling",
"headpose",
"facequality",
"blur",
"eyestatus",
"emotion",
"ethnicity",
"beauty",
"mouthstatus",
"eyegaze",
"skinstatus"
],
/** face++ table version */
facePPDbVersion: number,
/** create face++ table sql. */
facePPDbCreateTableSql: string,
/** create face++ index sqls. */
facePPDbCreateIndexSqls: string[],
/** face check backend */
faceBackEnd: "RudeCarnie" | "face-api.js" | "facepp",
/** nsfw check backend */
nsfwBackEnd: "NSFWJS" | "OpenNSFW",
/** NsfwJs judge function. Return true if accepted. */
nsfwJsJudgeFunction: (NsfwJsResult[]) => boolean,
/** NsfwJs table name */
nsfwJsDbTableName: string,
/** NsfwJs table version */
nsfwJsDbVersion: number,
/**
* Store prediction results in database.
* Currently supported: NSFWJS
*/
savePredictionResults: boolean,
/** tfjs backend */
tfjsBackEnd: "gpu" | "cpu",
/** entry point of face spinner api */
faceSpinnerApi: string[],
/** use face-api model */
faceApiUseModels: FaceApiModelName[],
/** face-api model directory base path */
faceApiModelBasePath: string,
/** face-api model urls */
faceApiModelUrlsByName: {
[FaceApiModelName]: string[]
},
/** create NSFWJS table sql. */
nsfwJsDbCreateTableSql: string,
/** create NSFWJS index sqls. */
nsfwJsDbCreateIndexSqls: string[],
/** return true: skip next process, return false: process next step */
nsfwPostJudgeFunction: FileInfo => boolean,
/** entry point of nsfw api */
nsfwApi: string[],
/** entry point of face gender detect api */
faceDetectWithGenderApi: string[],
/** entry point of face age detect api */
facePredictAgeApi: string[],
/** If set to true, files rejected by deep learning judgment are immediately deleted. */
instantDelete: boolean,
/**
* Judge by "and", "or" or "in" multiple deep learning judgments.
* If "and" it is necessary to clear all judgments, "or" should clear any judgment.
*/
logicalOperation: DeepLearningLogicalOperation,
/** "nsfw" or "sfw". */
nsfwType: NsfwType,
/** "allow" or "disallow" or "none". */
nsfwMode: DeepLearningMode,
/** isAcceptable for nsfwMode = "none" */
nsfwModeNoneDefault: true,
/**
* Threshold used for OpenNSFW judgment.
* Those that exceeded this threshold are "allow" or "disallow".
*/
nsfwThreshold: number,
/** Age and gender to use for judgment. All others are ignored. */
faceCategories: [GenderClass, AgeClass][],
/** "allow" or "disallow". */
faceMode: DeepLearningMode,
/** isAcceptable for faceMode = "none" */
faceModeNoneDefault: true,
/** Faces of smaller long side pixels are ignored. */
faceMinLongSide: number
};
export type DefaultConfig = {
recovery: boolean,
amazonDriveApiUrl: string,
amazonLoginUrl: string,
amazonDriveBaseDir: string,
amazonUser: string,
amazonPassword: string,
amazonBaseDir: string,
serverHttpsPort: number,
serverPort: number,
channelDbName: string,
channelDbCreateTableSql: string,
tagDbName: string,
tagDbLength: number,
tagDbCreateTableSql: string,
tagDbCreateIndexSqls: string[],
processStateSkipFunction: HashRow => boolean,
processStateDbName: string,
processStateDbCreateTableSql: string,
processStateDbCreateIndexSqls: string[],
deleteMode: DeleteModeType,
noTransfer: boolean,
archiveExtract: boolean,
archiveExtractCommand: string,
cacheVersion: number,
sortMarksFunction: (ReasonType, FileInfo) => string[],
deepLearningConfig: DeepLearningConfig,
instantDelete: boolean,
useFileName: boolean,
fileNameWhiteList: (string | RegExp)[],
useImageMagickHash: boolean,
log4jsConfig: Object,
dummyPath: string,
maxWorkers: number,
maxCpuLoadPercent: number,
hashAlgorithm: string,
defaultLogLevel: string,
dbBasePath: string,
dbTableName: string,
dbCreateTableSql: string,
dbCreateIndexSqls: string[],
dHashExactThreshold: number,
ignoreVideoDamage: boolean,
ignoreAudioDamage: boolean,
pHashIgnoreSameDir: boolean,
pHashExactThreshold: number,
pHashSearchThreshold: number,
pHashSearchRatioRangeOffset: number,
meanExactThreshold: number,
relativeResolutionRatioThreshold: number,
relativeFileSizeRatioThreshold: number,
forceTransfer: boolean,
renameRules: ([string | RegExp, string] | (string => string))[],
ngDirPathPatterns: (string | RegExp)[],
ngFileNamePatterns: (string | RegExp)[],
baseLibraryPathByType: {
[ClassifyType]: string
},
libraryPathDate?: Date,
libraryPathHourOffset: number,
minFileSizeByType: {
[ClassifyType]: number
},
minResolutionByType: {
[ClassifyType]: number
},
minLongSideByType: {
[ClassifyType]: number
},
classifyTypeByExtension: {
[string]: ClassifyType
}
};
/** CLI options */
export type CommanderConfig = {
/** https server mode port number. */
serverHttpsPort?: number,
/** server mode port number. */
serverPort?: number,
/** server mode. */
server?: boolean,
/** reset face-api model */
resetFaceApiModel?: boolean,
/** acd sync lmit. */
acdSync?: number,
/** db fill lmit. */
dbFill?: number,
/** db repair mode. */
dbRepair?: boolean,
/**
* manual mode.
* The file is not moved and the current path is registered in the destination.
*/
manual?: boolean,
/**
* keep mode.
* The file to be processed is treated as "keeping".
* These are preferential treatment at the time of duplication determination.
*/
keep?: boolean,
/** Wait at the end of processing. */
wait?: boolean,
/** prevent log output */
quiet?: boolean,
/** relocate mode. Relocate already imported files. */
relocate?: boolean,
/** Output processing result report to stdout. */
report: boolean,
/** debug log */
verbose?: boolean,
/** log level for log4js */
logLevel?: string,
/** use log config */
logConfig: boolean,
/** target file or folder path. */
path?: string,
/** Use pHash duplication judgment? */
pHash: boolean,
/** Cache information gathered from files. */
cache: boolean,
/** Use the directory path where the current file exists for the destination directory path? */
dirKeep: boolean,
/**
* Do not process files.
* Read operations, processes that do not affect the file itself are performed.
*/
dryrun: ?boolean,
/**
* sweep mode
*/
sweep: ?boolean
};
export type ForceConfig = {
manual?: boolean,
keep?: boolean,
report?: boolean,
pHash?: boolean,
dirKeep?: boolean,
cache?: boolean,
dryrun?: boolean
};
export type UserBaseConfig = {
/** recovery mode */
recovery?: boolean,
/** amazon drive api url */
amazonDriveApiUrl?: string,
/** amazon login url */
amazonLoginUrl?: string,
/** amazon backup base directory */
amazonDriveBaseDir?: string,
/** amazon cloud drive account. */
amazonUser?: string,
/** amazon cloud drive password. */
amazonPassword?: string,
/** amazon cloud drive config dir. */
amazonBaseDir?: string,
/** server mode port number. */
serverPort?: number,
/** process state table name */
processStateDbName?: string,
/** process state create table sql. */
processStateDbCreateTableSql?: string,
/** process state create index sqls. */
processStateDbCreateIndexSqls?: string[],
/** no "transfer" move. */
noTransfer?: boolean,
/** delete mode */
deleteMode?: DeleteModeType,
/** append additional file name prefix for examination. */
sortMarksFunction?: (ReasonType, FileInfo) => string[],
/** deep learning config */
deepLearningConfig?: DeepLearningConfig,
forceConfig?: ForceConfig,
/** If set to true, The archive file will be extracted. */
archiveExtract?: boolean,
/** archive extract command line */
archiveExtractCommand?: string,
/** If set to true, files rejected by judgment are immediately deleted. */
instantDelete?: boolean,
/** If true, use file name to check for identity. */
useFileName?: boolean,
/** white list for "useFileName" */
fileNameWiteList?: (string | RegExp)[],
/** If true, use ImageMagick's hash value to check for identity. */
useImageMagickHash?: boolean,
log4jsConfig?: Object,
/** When cpu load exceeds this value, dedupper will wait to process the next file. */
maxCpuLoadPercent?: number,
/** Number of concurrent executions. Used when processing a folder. */
maxWorkers?: number,
/**
* hash algorithm.
* The files having the same calculated value are handled as the same file.
*/
hashAlgorithm?: "md5" | "sha1" | "sha224" | "sha256" | "sha384" | "sha512",
/** log level for log4js */
defaultLogLevel?: "trace" | "debug" | "info" | "warn" | "error" | "fatal",
/** sqlite file folder path */
dbBasePath?: string,
/** table name */
dbTableName?: string,
/** create table sql. */
dbCreateTableSql?: string,
/** create index sqls. */
dbCreateIndexSqls?: string[],
/**
* If the Hamming distance of dHash is less than the threshold value,
* it is considered as the same image.
* Image file below the values of pHashExactThreshold and dHashExactThreshold,
* the file operation is immediately executed based on the judgment result.
*/
dHashExactThreshold?: number,
/**
* Ignore video file error.
*/
ignoreVideoDamage?: boolean,
/**
* Ignore audio file error.
*/
ignoreAudioDamage?: boolean,
/**
* Do not compare pHash between images in the same folder.
* Includes image files that once existed in the same folder.
*/
pHashIgnoreSameDir?: boolean,
/**
* If the Hamming distance of pHash is less than the threshold value,
* it is considered as the same image.
* Image file below the values of pHashExactThreshold and dHashExactThreshold,
* the file operation is immediately executed based on the judgment result.
*/
pHashExactThreshold?: number,
/**
* An image with Hamming distance below this threshold will be the same image candidate.
*/
pHashSearchThreshold?: number,
/**
* Permissible difference of image aspect ratio of files to be searched for pHash.
*/
pHashSearchRatioRangeOffset?: number,
/**
* Threshold value of average color value.
* Images that differ beyond this are not regarded as identical images.
*/
meanExactThreshold?: number,
/** Images whose resolution is lower than the ratio threshold are considered to be inferior. */
relativeResolutionRatioThreshold?: number,
/** Images whose file size is smaller than the ratio threshold are regarded as inferior. */
relativeFileSizeRatioThreshold?: number,
/** "replace" action will process as "transfer" action. */
forceTransfer?: boolean,
/** Rule of rename. Includes directory path. */
renameRules?: ([string | RegExp, string] | (string => string))[],
/** Delete the files in the directory matching this pattern. */
ngDirPathPatterns?: (string | RegExp)[],
/** Delete the file whose name matches this pattern. */
ngFileNamePatterns?: (string | RegExp)[],
/** The base destination for each type. */
baseLibraryPathByType?: {
[ClassifyType]: string
},
/** hour offset for library path date */
libraryPathHourOffset?: number,
/** manual library path date */
libraryPathDate?: Date,
/** Min file size for each type. */
minFileSizeByType?: {
[ClassifyType]: number
},
/** Min resolution each type. (pixel amount) */
minResolutionByType?: {
[ClassifyType]: number
},
/** Min long side each type. (pixel) */
minLongSideByType?: {
[ClassifyType]: number
},
/** What extension is what type? */
classifyTypeByExtension?: {
[string]: ClassifyType
}
};
/** Configure to apply to directories matching a specific pattern. */
export type PathMatchConfig = { [string]: UserBaseConfig };
/** Configure to apply to specific classify type file. */
export type ClassifyTypeConfig = { [ClassifyType]: UserBaseConfig };
/** user config */
export type UserConfig = UserBaseConfig & {
pathMatchConfig?: PathMatchConfig
};
export type NsfwJsHashRow = {
hash: string,
neutral: number,
drawing: number,
hentai: number,
porn: number,
sexy: number,
porn_sexy: number,
hentai_porn_sexy: number,
hentai_sexy: number,
hentai_porn: number,
version: number
};
export type FacePPRow = {
landmark: string,
image_id: string,
hash: string,
face_token: string,
face_num: number,
version: number,
emotion_sadness: number,
emotion_neutral: number,
emotion_disgust: number,
emotion_anger: number,
emotion_surprise: number,
emotion_fear: number,
emotion_happiness: number,
beauty_female_score: number,
beauty_male_score: number,
gender: FacePPGender,
age: number,
mouth_close: number,
mouth_surgical_mask_or_respirator: number,
mouth_open: number,
mouth_other_occlusion: number,
glass: FacePPGlass,
skin_dark_circle: number,
skin_stain: number,
skin_acne: number,
skin_health: number,
headpose_status: number,
headpose_yaw_angle: number,
headpose_pitch_angle: number,
headpose_roll_angle: number,
gaussianblur: number,
motionblur: number,
blurness: number,
smile: number,
eye_status_left_normal_glass_eye_open: number,
eye_status_left_normal_glass_eye_close: number,
eye_status_left_no_glass_eye_close: number,
eye_status_left_no_glass_eye_open: number,
eye_status_left_occlusion: number,
eye_status_left_dark_glasses: number,
eye_status_right_normal_glass_eye_open: number,
eye_status_right_normal_glass_eye_close: number,
eye_status_right_no_glass_eye_close: number,
eye_status_right_no_glass_eye_open: number,
eye_status_right_occlusion: number,
eye_status_right_dark_glasses: number,
eyegaze_right_position_x_coordinate: number,
eyegaze_right_position_y_coordinate: number,
eyegaze_right_vector_z: number,
eyegaze_right_vector_x: number,
eyegaze_right_vector_y: number,
eyegaze_left_position_x_coordinate: number,
eyegaze_left_position_y_coordinate: number,
eyegaze_left_vector_z: number,
eyegaze_left_vector_x: number,
eyegaze_left_vector_y: number,
facequality: number,
ethnicity: string,
eye_gaze_status: number,
top: number,
left: number,
width: number,
height: number
};
export type ImageContentsInfo = {
hash: string,
width: number,
height: number,
ratio: number,
damaged: boolean
};
export type Config = DefaultConfig &
CommanderConfig & {
forceConfig?: ForceConfig,
pathMatchConfig?: PathMatchConfig,
classifyTypeConfig?: ClassifyTypeConfig,
getLogger: Object => Logger
};