includes/media/DjVuImage.php
<?php
/**
* DjVu image handler.
*
* Copyright © 2006 Brooke Vibber <bvibber@wikimedia.org>
* https://www.mediawiki.org/
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup Media
*/
use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use MediaWiki\Shell\Shell;
use Wikimedia\AtEase\AtEase;
/**
* Support for detecting/validating DjVu image files and getting
* some basic file metadata (resolution etc)
*
* File format docs are available in source package for DjVuLibre:
* http://djvulibre.djvuzone.org/
*
* @ingroup Media
*/
class DjVuImage {
/**
* Memory limit for the DjVu description software
*/
private const DJVUTXT_MEMORY_LIMIT = 300_000_000;
/** @var string */
private $mFilename;
/**
* @param string $filename The DjVu file name.
*/
public function __construct( $filename ) {
$this->mFilename = $filename;
}
/**
* Check if the given file is indeed a valid DjVu image file
* @return bool
*/
public function isValid() {
$info = $this->getInfo();
return $info !== false;
}
/**
* Return width and height
* @return array An array with "width" and "height" keys, or an empty array on failure.
*/
public function getImageSize() {
$data = $this->getInfo();
if ( $data !== false ) {
return [
'width' => $data['width'],
'height' => $data['height']
];
}
return [];
}
// ---------
/**
* For debugging; dump the IFF chunk structure
*/
public function dump() {
$file = fopen( $this->mFilename, 'rb' );
$header = fread( $file, 12 );
$arr = unpack( 'a4magic/a4chunk/NchunkLength', $header );
$chunk = $arr['chunk'];
$chunkLength = $arr['chunkLength'];
echo "$chunk $chunkLength\n";
$this->dumpForm( $file, $chunkLength, 1 );
fclose( $file );
}
private function dumpForm( $file, $length, $indent ) {
$start = ftell( $file );
$secondary = fread( $file, 4 );
echo str_repeat( ' ', $indent * 4 ) . "($secondary)\n";
while ( ftell( $file ) - $start < $length ) {
$chunkHeader = fread( $file, 8 );
if ( $chunkHeader == '' ) {
break;
}
$arr = unpack( 'a4chunk/NchunkLength', $chunkHeader );
$chunk = $arr['chunk'];
$chunkLength = $arr['chunkLength'];
echo str_repeat( ' ', $indent * 4 ) . "$chunk $chunkLength\n";
if ( $chunk === 'FORM' ) {
$this->dumpForm( $file, $chunkLength, $indent + 1 );
} else {
fseek( $file, $chunkLength, SEEK_CUR );
if ( $chunkLength & 1 ) {
// Padding byte between chunks
fseek( $file, 1, SEEK_CUR );
}
}
}
}
private function getInfo() {
AtEase::suppressWarnings();
$file = fopen( $this->mFilename, 'rb' );
AtEase::restoreWarnings();
if ( $file === false ) {
wfDebug( __METHOD__ . ": missing or failed file read" );
return false;
}
$header = fread( $file, 16 );
$info = false;
if ( strlen( $header ) < 16 ) {
wfDebug( __METHOD__ . ": too short file header" );
} else {
$arr = unpack( 'a4magic/a4form/NformLength/a4subtype', $header );
$subtype = $arr['subtype'];
if ( $arr['magic'] !== 'AT&T' ) {
wfDebug( __METHOD__ . ": not a DjVu file" );
} elseif ( $subtype === 'DJVU' ) {
// Single-page document
$info = $this->getPageInfo( $file );
} elseif ( $subtype === 'DJVM' ) {
// Multi-page document
$info = $this->getMultiPageInfo( $file, $arr['formLength'] );
} else {
wfDebug( __METHOD__ . ": unrecognized DJVU file type '{$arr['subtype']}'" );
}
}
fclose( $file );
return $info;
}
private function readChunk( $file ) {
$header = fread( $file, 8 );
if ( strlen( $header ) < 8 ) {
return [ false, 0 ];
}
$arr = unpack( 'a4chunk/Nlength', $header );
return [ $arr['chunk'], $arr['length'] ];
}
private function skipChunk( $file, $chunkLength ) {
fseek( $file, $chunkLength, SEEK_CUR );
if ( ( $chunkLength & 1 ) && !feof( $file ) ) {
// padding byte
fseek( $file, 1, SEEK_CUR );
}
}
private function getMultiPageInfo( $file, $formLength ) {
// For now, we'll just look for the first page in the file
// and report its information, hoping others are the same size.
$start = ftell( $file );
do {
[ $chunk, $length ] = $this->readChunk( $file );
if ( !$chunk ) {
break;
}
if ( $chunk === 'FORM' ) {
$subtype = fread( $file, 4 );
if ( $subtype === 'DJVU' ) {
wfDebug( __METHOD__ . ": found first subpage" );
return $this->getPageInfo( $file );
}
$this->skipChunk( $file, $length - 4 );
} else {
wfDebug( __METHOD__ . ": skipping '$chunk' chunk" );
$this->skipChunk( $file, $length );
}
} while ( $length != 0 && !feof( $file ) && ftell( $file ) - $start < $formLength );
wfDebug( __METHOD__ . ": multi-page DJVU file contained no pages" );
return false;
}
private function getPageInfo( $file ) {
[ $chunk, $length ] = $this->readChunk( $file );
if ( $chunk !== 'INFO' ) {
wfDebug( __METHOD__ . ": expected INFO chunk, got '$chunk'" );
return false;
}
if ( $length < 9 ) {
wfDebug( __METHOD__ . ": INFO should be 9 or 10 bytes, found $length" );
return false;
}
$data = fread( $file, $length );
if ( strlen( $data ) < $length ) {
wfDebug( __METHOD__ . ": INFO chunk cut off" );
return false;
}
$arr = unpack(
'nwidth/' .
'nheight/' .
'Cminor/' .
'Cmajor/' .
'vresolution/' .
'Cgamma', $data );
# Newer files have rotation info in byte 10, but we don't use it yet.
return [
'width' => $arr['width'],
'height' => $arr['height'],
'version' => "{$arr['major']}.{$arr['minor']}",
'resolution' => $arr['resolution'],
'gamma' => $arr['gamma'] / 10.0 ];
}
/**
* Return an array describing the DjVu image
* @return array|null|false
*/
public function retrieveMetaData() {
$config = MediaWikiServices::getInstance()->getMainConfig();
$djvuDump = $config->get( MainConfigNames::DjvuDump );
$djvuTxt = $config->get( MainConfigNames::DjvuTxt );
$djvuUseBoxedCommand = $config->get( MainConfigNames::DjvuUseBoxedCommand );
$shell = $config->get( MainConfigNames::ShellboxShell );
if ( !$this->isValid() ) {
return false;
}
if ( $djvuTxt === null && $djvuDump === null ) {
return [];
}
$txt = null;
$dump = null;
if ( $djvuUseBoxedCommand ) {
$command = MediaWikiServices::getInstance()->getShellCommandFactory()
->createBoxed( 'djvu' )
->disableNetwork()
->firejailDefaultSeccomp()
->routeName( 'djvu-metadata' )
->params( $shell, 'scripts/retrieveDjvuMetaData.sh' )
->inputFileFromFile(
'scripts/retrieveDjvuMetaData.sh',
__DIR__ . '/scripts/retrieveDjvuMetaData.sh' )
->inputFileFromFile( 'file.djvu', $this->mFilename )
->memoryLimit( self::DJVUTXT_MEMORY_LIMIT );
$env = [];
if ( $djvuDump !== null ) {
$env['DJVU_DUMP'] = $djvuDump;
$command->outputFileToString( 'dump' );
}
if ( $djvuTxt !== null ) {
$env['DJVU_TXT'] = $djvuTxt;
$command->outputFileToString( 'txt' );
}
$result = $command
->environment( $env )
->execute();
if ( $result->getExitCode() !== 0 ) {
wfDebug( 'retrieveDjvuMetaData failed with exit code ' . $result->getExitCode() );
return false;
}
if ( $djvuDump !== null ) {
if ( $result->wasReceived( 'dump' ) ) {
$dump = $result->getFileContents( 'dump' );
} else {
wfDebug( __METHOD__ . ": did not receive dump file" );
}
}
if ( $djvuTxt !== null ) {
if ( $result->wasReceived( 'txt' ) ) {
$txt = $result->getFileContents( 'txt' );
} else {
wfDebug( __METHOD__ . ": did not receive text file" );
}
}
} else { // No boxedcommand
if ( $djvuDump !== null ) {
# djvudump is faster than djvutoxml (now abandoned) as of version 3.5
# https://sourceforge.net/p/djvu/bugs/71/
$cmd = Shell::escape( $djvuDump ) . ' ' . Shell::escape( $this->mFilename );
$dump = wfShellExec( $cmd );
}
if ( $djvuTxt !== null ) {
$cmd = Shell::escape( $djvuTxt ) . ' --detail=page ' . Shell::escape( $this->mFilename );
wfDebug( __METHOD__ . ": $cmd" );
$retval = 0;
$txt = wfShellExec( $cmd, $retval, [], [ 'memory' => self::DJVUTXT_MEMORY_LIMIT ] );
if ( $retval !== 0 ) {
$txt = null;
}
}
}
# Convert dump to array
$json = [];
if ( $dump !== null ) {
$data = $this->convertDumpToJSON( $dump );
if ( $data !== false ) {
$json = [ 'data' => $data ];
}
}
# Text layer
if ( $txt !== null ) {
# Strip some control characters
# Ignore carriage returns
$txt = preg_replace( "/\\\\013/", "", $txt );
# Replace runs of OCR region separators with a single extra line break
$txt = preg_replace( "/(?:\\\\(035|037))+/", "\n", $txt );
$reg = <<<EOR
/\(page\s[\d-]*\s[\d-]*\s[\d-]*\s[\d-]*\s*"
((?> # Text to match is composed of atoms of either:
\\\\. # - any escaped character
| # - any character different from " and \
[^"\\\\]+
)*?)
"\s*\)
| # Or page can be empty ; in this case, djvutxt dumps ()
\(\s*()\)/sx
EOR;
$matches = [];
preg_match_all( $reg, $txt, $matches );
$json['text'] = array_map( [ $this, 'pageTextCallback' ], $matches[1] );
} else {
$json['text'] = [];
}
return $json;
}
private function pageTextCallback( string $match ) {
# Get rid of invalid UTF-8
$val = UtfNormal\Validator::cleanUp( stripcslashes( $match ) );
return str_replace( '�', '', $val );
}
/**
* @param string $dump
* @return array|false
*/
private function convertDumpToJSON( $dump ) {
if ( strval( $dump ) == '' ) {
return false;
}
$dump = str_replace( "\r", '', $dump );
$line = strtok( $dump, "\n" );
$m = false;
$good = false;
$result = [];
if ( preg_match( '/^( *)FORM:DJVU/', $line, $m ) ) {
# Single-page
$parsed = $this->parseFormDjvu( $line );
if ( $parsed ) {
$good = true;
} else {
return false;
}
$result['pages'] = [ $parsed ];
} elseif ( preg_match( '/^( *)FORM:DJVM/', $line, $m ) ) {
# Multi-page
$parentLevel = strlen( $m[1] );
# Find DIRM
$line = strtok( "\n" );
$result['pages'] = [];
while ( $line !== false ) {
$childLevel = strspn( $line, ' ' );
if ( $childLevel <= $parentLevel ) {
# End of chunk
break;
}
if ( preg_match( '/^ *DIRM.*indirect/', $line ) ) {
wfDebug( "Indirect multi-page DjVu document, bad for server!" );
return false;
}
if ( preg_match( '/^ *FORM:DJVU/', $line ) ) {
# Found page
$parsed = $this->parseFormDjvu( $line );
if ( $parsed ) {
$good = true;
} else {
return false;
}
$result['pages'][] = $parsed;
}
$line = strtok( "\n" );
}
}
if ( !$good ) {
return false;
}
return $result;
}
private function parseFormDjvu( $line ) {
$parentLevel = strspn( $line, ' ' );
$line = strtok( "\n" );
# Find INFO
while ( $line !== false ) {
$childLevel = strspn( $line, ' ' );
if ( $childLevel <= $parentLevel ) {
# End of chunk
break;
}
if ( preg_match(
'/^ *INFO *\[\d*] *DjVu *(\d+)x(\d+), *\w*, *(\d+) *dpi, *gamma=([0-9.-]+)/',
$line,
$m
) ) {
return [
'height' => (int)$m[2],
'width' => (int)$m[1],
'dpi' => (float)$m[3],
'gamma' => (float)$m[4],
];
}
$line = strtok( "\n" );
}
# Not found
return false;
}
}