superdesk/superdesk-client-core

View on GitHub
scripts/extensions/videoEditor/src/VideoEditor.tsx

Summary

Maintainability
F
4 days
Test Coverage
import * as React from 'react';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import ReactCrop from 'react-image-crop';
import 'react-image-crop/dist/ReactCrop.css';
import {ISuperdesk, IArticle} from 'superdesk-api';
import {omit, isEqual} from 'lodash';

import {VideoEditorTools} from './VideoEditorTools';
import {VideoTimeline} from './VideoTimeline/VideoTimeline';
import {VideoEditorHeader} from './VideoEditorHeader';
import {VideoEditorThumbnail} from './VideoEditorThumbnail';
import {IThumbnail, ICrop, IErrorMessage, ITimelineThumbnail, IVideoProject} from './interfaces';

interface IProps {
    article: IArticle;
    superdesk: ISuperdesk;
    onClose: () => void;
}

interface IState {
    transformations: {
        trim: {
            start: number;
            end: number;
        };
        crop: ICrop;
        quality: number;
        degree: number;
    };
    thumbnails: Array<IThumbnail>;
    loading: {
        video: boolean;
        thumbnail: boolean;
    };
    loadingText: string;
    scale: number;
    videoSrc: string;
    article: IArticle;
    cropEnabled: boolean;
    playing: boolean;
    prevVideoHeight: number;
}

interface IVideoEditParams {
    crop?: string;
    rotate?: number;
    trim?: string;
    scale?: number;
}

const initTransformations: IState['transformations'] = {
    crop: {aspect: 16 / 9, unit: 'px', width: 0, height: 0, x: 0, y: 0},
    degree: 0,
    trim: {start: 0, end: 0},
    quality: 0,
};

export class VideoEditor extends React.Component<IProps, IState> {
    private videoRef: React.RefObject<HTMLVideoElement>;
    private reactCropRef: React.RefObject<ReactCrop>;
    private reactCropInitImage: string;
    private intervalThumbnails: number;
    private intervalCheckVideo: number;
    private hasTransitionRun: boolean;
    private videoWrapper: HTMLDivElement | null;
    private videoContainerSize: number;
    private videoToolsSize: number;

    constructor(props: IProps) {
        super(props);
        this.videoRef = React.createRef();
        this.reactCropRef = React.createRef();
        this.intervalThumbnails = 0;
        this.intervalCheckVideo = 0;
        this.state = {
            transformations: initTransformations,
            cropEnabled: false,
            playing: false,
            loading: {
                video: false,
                thumbnail: false,
            },
            loadingText: '',
            scale: 1,
            prevVideoHeight: 1,
            thumbnails: [],
            videoSrc: '',
            article: this.props.article,
        };
        this.videoWrapper = null;
        this.videoContainerSize = 0;
        this.videoToolsSize = 0;
        this.hasTransitionRun = true;
        this.reactCropInitImage = '';

        this.handleClose = this.handleClose.bind(this);
        this.handleTrim = this.handleTrim.bind(this);
        this.handleRotate = this.handleRotate.bind(this);
        this.handleRotateTransitionEnd = this.handleRotateTransitionEnd.bind(this);
        this.handleCrop = this.handleCrop.bind(this);
        this.handleToggleLoading = this.handleToggleLoading.bind(this);
        this.handleToggleCrop = this.handleToggleCrop.bind(this);
        this.playPause = this.playPause.bind(this);
        this.handleQualityChange = this.handleQualityChange.bind(this);
        this.handleSave = this.handleSave.bind(this);
        this.getCropRotate = this.getCropRotate.bind(this);
        this.setVideoWrapper = this.setVideoWrapper.bind(this);
        this.setVideoToolsSize = this.setVideoToolsSize.bind(this);
        this.setVideoContainerSize = this.setVideoContainerSize.bind(this);
        this.showErrorMessage = this.showErrorMessage.bind(this);
        this.checkIsDirty = this.checkIsDirty.bind(this);
    }

    componentDidMount() {
        const canvas = document.createElement('canvas');
        const ctx = canvas.getContext('2d');

        // initilize image for React Crop
        // this is the boudary of crop area, user can't move or draw crop outside of this so image need
        // to be big enough to cover entire video even once rotated
        canvas.width = 2000;
        canvas.height = 2000;
        if (ctx == null) {
            return;
        }

        ctx.globalAlpha = 0;
        ctx.fillStyle = 'rgba(0, 0, 200, 0.5)';
        const {gettext} = this.props.superdesk.localization;

        this.handleToggleLoading(true, gettext('Loading video...'));
        this.handleCheckingVideo(false, () => {
            const text = gettext('Video is editing, please wait...');

            if (this.state.loadingText !== text) {
                this.setState({
                    loadingText: text,
                });
            }
        });
        this.reactCropInitImage = canvas.toDataURL();
    }

    componentWillUnmount() {
        clearInterval(this.intervalThumbnails);
        clearInterval(this.intervalCheckVideo);
    }

    handleClose() {
        this.props.onClose();

        this.props.superdesk.entities.article.patch(
            this.state.article,
            {
                renditions: this.state.article.renditions,
            },
            {patchDirectlyAndOverwriteAuthoringValues: true},
        );
    }

    handleCheckingVideo(resetState: boolean = true, callback?: () => void) {
        this.intervalCheckVideo = window.setInterval(() => {
            this.props.superdesk.dataApi
                .findOne<IVideoProject>('video_edit', this.state.article._id)
                .then((result) => {
                    const processing = result.project.processing;

                    if (processing.video === false && processing.thumbnail_preview === false) {
                        const state = resetState ? this.getResetState() : {};

                        clearInterval(this.intervalCheckVideo);
                        this.setState({
                            ...state,
                            loading: {
                                thumbnail: false,
                                video: false,
                            },
                            thumbnails: [],
                            // force browser to reload video duration after trimming video
                            // because if video src does not change, browser will not update video
                            // duration even after caching is disabled
                            videoSrc: result.renditions?.original?.href + `?t=${Math.random()}`,
                            article: {
                                ...this.state.article,
                                renditions: result.renditions,
                            },
                        });
                        this.loadTimelineThumbnails();
                    } else {
                        callback?.();
                    }
                })
                .catch((err: IErrorMessage) => {
                    this.showErrorMessage(err);
                    clearInterval(this.intervalCheckVideo);
                });
        }, 3000);
    }

    handleTrim(start: number, end: number) {
        this.setState({
            transformations: {
                ...this.state.transformations,
                trim: {
                    start: start,
                    end: end,
                },
            },
        });
    }

    handleRotate() {
        const video = this.videoRef.current;

        if (video == null) {
            return;
        }

        this.hasTransitionRun = false;
        const {getClass} = this.props.superdesk.utilities.CSS;
        const classList = video.classList;

        if (!classList.contains(getClass('rotate__transition'))) {
            classList.add(getClass('rotate__transition'));
        }
        const cropRef = this.reactCropRef.current?.['componentRef'];

        if (cropRef) {
            cropRef.style.visibility = 'hidden';
        }

        const refValue = video.getBoundingClientRect();
        const videoHeight = this.state.transformations.degree % 180 === -90 ? refValue.width : refValue.height;

        this.setState((prevState) => ({
            transformations: {
                ...this.state.transformations,
                degree: prevState.transformations.degree - 90,
            },
            scale: 1,
            prevVideoHeight: this.getScale() * (videoHeight ?? 1),
        }));
    }

    handleRotateTransitionEnd() {
        // avoid transition rerun after set scale
        if (this.hasTransitionRun === true || this.videoRef.current == null) {
            return;
        }
        this.hasTransitionRun = true;
        const {getClass} = this.props.superdesk.utilities.CSS;
        const degree = this.state.transformations.degree % 360 === 0 ? 0 : this.state.transformations.degree;

        // avoid running transition on setting 360 degree to 0
        if (degree === 0) {
            this.videoRef.current.classList.remove(getClass('rotate__transition'));
        }
        const scale = this.getScale();

        let crop = this.state.transformations.crop;

        if (this.state.cropEnabled) {
            const refValue = this.videoRef.current.getBoundingClientRect();
            const videoHeight = this.state.transformations.degree % 180 === 0 ? refValue.height : refValue.width;
            // Crop is scaled along with video when it is first toggled
            // so we need to calculate ratio between video size before and after rotated
            // to scale crop appropriately
            const currentVideoHeight = scale * videoHeight;
            const delta = currentVideoHeight / this.state.prevVideoHeight;

            crop = {
                ...crop,
                aspect: 1 / (crop.aspect ?? 1),
                x: crop.y * delta,
                y: videoHeight - (crop.x + crop.width) * delta,
                height: crop.width * delta,
                width: crop.height * delta,
            };

            const cropRef = this.reactCropRef.current?.['componentRef'];

            if (cropRef) {
                cropRef.style.visibility = 'unset';
            }
        }
        this.setState({transformations: {...this.state.transformations, degree: degree, crop: crop}, scale: scale});
    }

    handleCrop(newCrop: ICrop) {
        if (newCrop.x == null || newCrop.y == null || newCrop.width == null || newCrop.height == null) {
            throw new Error('Invalid state');
        }
        if (this.videoRef.current == null) {
            throw new Error('Could not load video');
        }

        newCrop.x = Math.floor(newCrop.x);
        newCrop.y = Math.floor(newCrop.y);
        newCrop.width = Math.floor(newCrop.width);
        newCrop.height = Math.floor(newCrop.height);

        // newCrop lost aspect when cropping while rotating sometimes
        if (!newCrop.aspect) {
            newCrop.aspect = this.state.transformations.crop.aspect;
        }

        // when first draw crop area, ReactImageCrop trigger a bulk of change event with the same
        // newCrop value, using throttle with value about 50 did not help much but increase interval
        // may result in lagging
        if (Object.values(this.state.transformations.crop).toString() === Object.values(newCrop).toString()) {
            return;
        }

        this.setState({transformations: {...this.state.transformations, crop: newCrop}});
    }

    playPause() {
        const videoElement = this.videoRef.current;

        if (videoElement == null) {
            return;
        }

        if (this.state.playing) {
            videoElement.pause();
        } else {
            // Video will play from the beginning after finished playing, but when it reaches trim end
            // play button will only play video a bit everytime it is clicked
            if (videoElement.currentTime > this.state.transformations.trim.end) {
                videoElement.currentTime = this.state.transformations.trim.start;
            }
            videoElement.play();
        }
    }

    handleToggleCrop(aspect: number) {
        const cropAspect = aspect || this.state.transformations.crop.aspect || 0;
        let crop = initTransformations.crop;

        if (this.state.cropEnabled === false) {
            crop = this.getInitialCropSize(cropAspect);
        }
        this.setState({
            transformations: {...this.state.transformations, crop: crop},
            cropEnabled: !this.state.cropEnabled,
        });
    }

    handleToggleLoading(isToggle: boolean, text: string = '', type: 'video' | 'thumbnail' = 'video') {
        if (this.state.playing) {
            this.playPause();
        }
        this.setState({
            loading: {
                ...this.state.loading,
                [type]: isToggle,
            },
            loadingText: text || this.state.loadingText,
        });
    }

    handleQualityChange(quality: number) {
        if (!this.videoRef.current) {
            return;
        }

        const {videoWidth, videoHeight} = this.videoRef.current;
        const ratio = videoWidth / videoHeight;
        // resolution is calculated based on video height but video server scales based on video width
        const newQuality = ratio > 1 ? quality * ratio : quality;

        this.setState({transformations: {...this.state.transformations, quality: Math.ceil(newQuality)}});
    }

    handleSave() {
        const {dataApi} = this.props.superdesk;
        const {gettext} = this.props.superdesk.localization;
        const {x, y, width, height} = this.state.transformations.crop;
        const crop = this.getCropRotate({x: x, y: y, width: width, height: height});
        const body: IVideoEditParams = {
            crop: Object.values(crop).join(','),
            rotate: this.state.transformations.degree,
            trim: Object.values(this.state.transformations.trim).join(','),
            scale: this.state.transformations.quality,
        };

        if (body.crop === '0,0,0,0') {
            delete body.crop;
        }
        if (body.rotate === 0) {
            delete body.rotate;
        }
        if (body.trim === `0,${this.videoRef.current?.duration}`) {
            delete body.trim;
        }
        if (body.scale === 0) {
            delete body.scale;
        }

        // video server will crop before scaling video down
        // calculate expected scale value based on crop width so that the result
        // will be the same as using scale then crop
        if (body.scale !== undefined && crop.width !== 0 && this.videoRef.current) {
            body.scale = Math.ceil((crop.width / this.videoRef.current.videoWidth) * body.scale);
        }

        if (Object.keys(body).length !== 0) {
            // Save will trigger another polling timeline thumbnails, so current interval id
            // will be lost, thus causing infinite loop polling interval if the current one
            // is still in progress
            clearInterval(this.intervalThumbnails);
            dataApi
                .create('video_edit', {
                    edit: body,
                    item: {
                        _id: this.state.article._id,
                        renditions: this.state.article.renditions,
                    },
                })
                .then(() => {
                    this.handleToggleLoading(true, gettext('Video is editing, please wait...'));
                    this.handleCheckingVideo();
                })
                .catch((err: IErrorMessage) => {
                    this.showErrorMessage(err);
                });
        }
    }

    checkIsDirty() {
        if (this.videoRef.current === null || this.state.transformations.trim.end === 0) {
            return false;
        }

        // ignore trim.end as initState don't load video duration due to videoRef can be null in componentDidMount
        // confirm bar should not be toggled when user change crop aspect
        if (
            this.state.transformations.trim.end !== this.videoRef.current.duration ||
            !isEqual(
                omit(this.state.transformations, ['trim.end', 'crop.aspect']),
                omit(initTransformations, ['trim.end', 'crop.aspect']),
            )
        ) {
            return true;
        }
        return false;
    }

    loadTimelineThumbnails() {
        this.intervalThumbnails = window.setInterval(() => {
            this.props.superdesk.dataApi
                .findOne<ITimelineThumbnail>('video_edit', this.state.article._id + '?action=timeline')
                .then((result: ITimelineThumbnail) => {
                    if (result.thumbnails.length > 0) {
                        clearInterval(this.intervalThumbnails);
                        this.setState({thumbnails: result.thumbnails});
                    }
                })
                .catch((err: IErrorMessage) => {
                    // conflict happens when user trigger another video edit right after finished one
                    // error should not be showed as this is not caused by user intentionally
                    if (err.internal_error !== 409) {
                        this.showErrorMessage(err);
                    }
                    clearInterval(this.intervalThumbnails);
                });
        }, 3000);
    }

    getResetState() {
        return {
            transformations: {
                ...initTransformations,
                trim: {
                    start: 0,
                    end: this.videoRef.current?.duration ?? 0,
                },
            },
            cropEnabled: false,
            scale: 1,
        };
    }

    // calculate crop real size as crop value is not based on real video size
    // but scaled video to fit into container
    getCropSize(video: HTMLVideoElement, inputCrop: ICrop): ICrop {
        let {width, height} = video.getBoundingClientRect();

        if (this.state.transformations.degree % 180 !== 0) {
            [width, height] = [height, width];
        }

        const crop = Object.assign({}, inputCrop);
        const scaleX = video.videoWidth / width;
        const scaleY = video.videoHeight / height;

        return {
            ...crop,
            x: Math.floor(crop.x * scaleX),
            y: Math.floor(crop.y * scaleY),
            width: Math.floor(crop.width * scaleX),
            height: Math.floor(crop.height * scaleY),
        };
    }

    // Set initial crop size to 80% of full video size to make the UX more user-friendly.
    // If it was set to 100%, moving the crop area might not be possible
    // and resize indicators at the corners might not be that clearly visible.
    getInitialCropSize(aspect: number, scale: number = 1) {
        if (this.videoRef.current == null) {
            throw new Error('Could not load video');
        }
        let {width, height} = this.videoRef.current.getBoundingClientRect();

        width = (width * scale * 80) / 100;
        height = (height * scale * 80) / 100;

        const ratio = width / height;

        if (ratio > 1) {
            width = height * aspect;
        } else {
            height = width * aspect;
        }
        return {
            ...initTransformations.crop,
            aspect: aspect,
            width: width,
            height: height,
        };
    }

    // get original crop value while rotating video
    // because server crops video before rotating
    getCropRotate(crop: ICrop): ICrop {
        if (this.videoRef.current == null) {
            throw new Error('Could not get rotated video crop value');
        }

        const {width: currentWidth, height: currentHeight} = this.videoRef.current.getBoundingClientRect();
        const rotate = this.state.transformations.degree;
        const {x, y, width, height} = crop;

        // do not use state cropEnabled, because user can toggle crop then capture without draw crop area
        if ([x, y, width, height].every((value) => value === 0) === true) {
            return this.getCropSize(this.videoRef.current, crop);
        }

        switch (rotate) {
        case -90:
            return this.getCropSize(
                this.videoRef.current,
                {
                    ...crop,
                    x: Math.abs(currentHeight - height - y),
                    y: x,
                    width: height,
                    height: width,
                },
            );
        case -180:
            return this.getCropSize(
                this.videoRef.current,
                {
                    ...crop,
                    x: Math.abs(currentWidth - (x + width)),
                    y: Math.abs(currentHeight - (y + height)),
                },
            );
        case -270:
            return this.getCropSize(
                this.videoRef.current,
                {
                    ...crop,
                    x: y,
                    y: Math.abs(currentWidth - width - x),
                    width: height,
                    height: width,
                },
            );
        default:
            return this.getCropSize(this.videoRef.current, crop);
        }
    }

    setVideoWrapper(element: HTMLDivElement) {
        if (element == null) {
            return;
        }
        this.videoWrapper = element;
    }

    setVideoToolsSize(element: HTMLDivElement) {
        if (element == null) {
            return;
        }
        const {height} = element.getBoundingClientRect();
        const {marginTop} = window.getComputedStyle(element);
        // the remain of margin bottom of video tools is too big and overflowed to timeline
        const margin = parseFloat(marginTop) * 2;

        this.videoToolsSize = height + margin;
    }

    setVideoContainerSize(element: HTMLDivElement) {
        if (element == null) {
            return;
        }
        const {marginTop} = window.getComputedStyle(element);

        this.videoContainerSize = parseFloat(marginTop);
    }

    getWrapperHeight() {
        if (this.videoWrapper == null) {
            return 0;
        }
        const {height} = this.videoWrapper.getBoundingClientRect();

        return height - this.videoToolsSize - this.videoContainerSize;
    }

    getScale(): number {
        const videoRef = this.videoRef.current;

        if (!videoRef || videoRef.videoHeight === 0 || this.state.transformations.degree === 0) {
            return 1;
        }

        const videoHeight = this.state.transformations.degree % 180 !== 0 ? videoRef.videoWidth : videoRef.videoHeight;
        const wrapperHeight = this.getWrapperHeight();
        const {height} = videoRef.getBoundingClientRect();
        // ensure video image quality is not broken when scaling up
        const vh = videoHeight < wrapperHeight ? videoHeight : wrapperHeight;
        // round approximate 1 value (e.g. 1.0059880239)
        // avoid running unnecessary transformation transition on resetting state when rotating 360 degree
        const scale = Math.trunc(vh / height * 100) / 100;

        if (scale === 1) {
            return scale;
        }
        return vh / height;
    }

    showErrorMessage(errorResponse: IErrorMessage) {
        const error = Object.values(errorResponse._message).reduce((acc, curr) => acc.concat(curr), []);

        this.props.superdesk.ui.alert(error.join('<br/>'));
    }

    render() {
        const {gettext} = this.props.superdesk.localization;
        const {getClass} = this.props.superdesk.utilities.CSS;
        const degree = this.state.transformations.degree + 'deg';
        const videoRef = this.videoRef.current;

        let width = 0,
            height = 0,
            videoHeight = 1;

        if (videoRef != null) {
            const scale = this.getScale();

            ({width, height} = videoRef.getBoundingClientRect());
            if (scale !== 1) {
                // video has not applied scale yet so ReactCrop will exceed video size
                width = width * scale;
                height = height * scale;
            }
            videoHeight = this.state.transformations.degree % 180 !== 0 ? videoRef.videoWidth : videoRef.videoHeight;
        }
        const isLoading = this.state.loading.video || this.state.loading.thumbnail;

        return (
            <div className="modal modal--fullscreen in" style={{zIndex: 1050, display: 'block'}}>
                <div className="modal__dialog">
                    <div className="modal__content">
                        <div className="modal__header modal__header--flex">
                            <h3 className="modal__heading">{gettext('Edit Video')}</h3>
                            <VideoEditorHeader
                                onClose={this.handleClose}
                                onReset={() => this.setState({...this.getResetState()})}
                                onSave={this.handleSave}
                                isDirty={this.checkIsDirty()}
                                isVideoLoading={this.state.loading.video}
                                gettext={gettext}
                            />
                        </div>
                        <div className="modal__body modal__body--no-padding">
                            {isLoading && (
                                <div className={getClass('loading')}>
                                    <div className={getClass('loading__text')}>{this.state.loadingText}</div>
                                </div>
                            )}
                            <div className="sd-photo-preview sd-photo-preview--edit-video">
                                <div className="sd-photo-preview__video" ref={this.setVideoWrapper}>
                                    <div className="sd-photo-preview__video-inner">
                                        <div
                                            className="sd-photo-preview__video-container"
                                            style={{marginBlockStart: '2rem'}}
                                            ref={this.setVideoContainerSize}
                                        >
                                            <video
                                                ref={this.videoRef}
                                                src={this.state.videoSrc}
                                                onPlay={() => this.setState({playing: true})}
                                                onPause={() => this.setState({playing: false})}
                                                onDurationChange={() => this.handleTrim(0, videoRef?.duration ?? 0)}
                                                style={{
                                                    transform: `rotate(${degree}) scale(${this.state.scale})`,
                                                    height: `${videoHeight}px`,
                                                    // chrome will add extra position for react crop if video has
                                                    // margin top, even if margin of those two are equal
                                                    marginBlockStart: 0,
                                                }}
                                                className={getClass('rotate__transition')}
                                                onTransitionEnd={this.handleRotateTransitionEnd}
                                                autoPlay
                                            />

                                            {this.state.cropEnabled && (
                                                <ReactCrop
                                                    ref={this.reactCropRef}
                                                    src={this.reactCropInitImage}
                                                    crop={this.state.transformations.crop}
                                                    keepSelection={true}
                                                    onChange={this.handleCrop}
                                                    className={getClass('crop')}
                                                    /* avoid limit height (auto) when video is portrait */
                                                    imageStyle={{maxWidth: 'unset'}}
                                                    style={{
                                                        width: width,
                                                        height: height,
                                                        background: 'unset',
                                                        position: 'absolute',
                                                    }}
                                                />
                                            )}
                                        </div>
                                        <VideoEditorTools
                                            wrapperRef={this.setVideoToolsSize}
                                            playPause={this.playPause}
                                            onRotate={this.handleRotate}
                                            onCrop={this.handleToggleCrop}
                                            onQualityChange={this.handleQualityChange}
                                            cropEnabled={this.state.cropEnabled}
                                            videoDegree={this.state.transformations.degree}
                                            videoPlaying={this.state.playing}
                                            videoQuality={this.state.transformations.quality}
                                            videoHeadline={this.state.article.headline}
                                            videoResolution={Math.min(
                                                videoRef?.videoWidth ?? 0,
                                                videoRef?.videoHeight ?? 0,
                                            )}
                                            gettext={gettext}
                                            getClass={getClass}
                                        />
                                    </div>
                                </div>
                                <div className="sd-photo-preview__thumb-strip sd-photo-preview__thumb-strip--video">
                                    {videoRef && (
                                        <>
                                            <VideoEditorThumbnail
                                                video={videoRef}
                                                article={this.state.article}
                                                onToggleLoading={this.handleToggleLoading}
                                                onSave={(renditions: IArticle['renditions'], etag: string) =>
                                                    // update preview thumbnail will change etag
                                                    this.setState({
                                                        article: {
                                                            ...this.state.article,
                                                            renditions: renditions,
                                                            _etag: etag,
                                                        }})
                                                }
                                                onError={this.showErrorMessage}
                                                crop={this.state.transformations.crop}
                                                rotate={this.state.transformations.degree}
                                                getCropRotate={this.getCropRotate}
                                                superdesk={this.props.superdesk}
                                            />
                                            <VideoTimeline
                                                video={videoRef}
                                                trim={this.state.transformations.trim}
                                                onTrim={this.handleTrim}
                                                thumbnails={this.state.thumbnails}
                                                getClass={getClass}
                                            />
                                        </>
                                    )}
                                </div>
                            </div>
                        </div>
                    </div>
                </div>
            </div>
        );
    }
}