superdesk/superdesk-client-core

View on GitHub
scripts/apps/ingest/directives/IngestSourcesContent.ts

Summary

Maintainability
F
1 wk
Test Coverage
import _, {throttle} from 'lodash';
import {cloneDeep} from 'lodash';
import {gettext} from 'core/utils';
import {appConfig} from 'appConfig';
import {IBaseRestApiResponse} from 'superdesk-api';
import {authenticateIngestProvider} from './authenticate-ingest-provider';
import {addWebsocketEventListener} from 'core/notification/notification';
import {notify} from 'core/notify/notify';

interface IFeedingServiceField {
    id?: string;
    type: string;
    label: string;
    readonly?: boolean;
    placeholder?: string;
    default_value?: string;
    errors?: { [key: number]: any };
    url?: string;
}

interface IProvider extends IBaseRestApiResponse {
    name: string;
    source: string;
    feeding_service: string;
    feed_parser?: string;
    content_types?: Array<any>;
    allow_remove_ingested?: boolean;
    disable_item_updates?: boolean;
    content_expiry?: number;
    config?: {};
    private?: {};
    ingested_count?: number;
    accepted_count?: number;
    token?: {};
    is_closed?: boolean;
    update_schedule?: {};
    idle_time?: {};
    last_updated?: string;
    last_ingested_id?: string;
    last_item_update?: string;
    rule_set?: any;
    routing_scheme?: any;
    notifications?: {};
    last_closed?: {};
    last_opened?: {};
    critical_errors?: {};
    skip_config_test?: boolean;
    url_id?: string;
}

IngestSourcesContent.$inject = ['ingestSources', 'api', '$location',
    'modal', '$filter', 'privileges'];

/**
 * @ngdoc directive
 * @module superdesk.apps.ingest
 * @name sdIngestSourcesContent
 *
 * @requires ingestSources
 * @requires notify
 * @requires api
 * @requires $location
 * @requires modal
 * @requires $filter
 * @requires config
 * @requires privileges
 *
 * @description Handles the management for Ingest Sources.
 */
export function IngestSourcesContent(ingestSources, api, $location,
    modal, $filter, privileges) {
    return {
        templateUrl: 'scripts/apps/ingest/views/settings/ingest-sources-content.html',
        link: function($scope) {
            $scope.waitForDirectiveReady = function() {
                return Promise.all([
                    ingestSources.fetchAllFeedingServicesAllowed(),
                ]);
            };

            $scope.getErrorMessage = (error) => {
                const msg = error?._error?.message ?? error?.message;

                return msg?.length > 0 ? msg : gettext('An error occured when testing config.');
            };

            $scope.waitForDirectiveReady().then((waitForDirectiveReadyResult) => {
                $scope.feedingServices = waitForDirectiveReadyResult[0];

                $scope.provider = null;
                $scope.allFeedParsers = [];
                $scope.feedParsers = [];
                $scope.fileTypes = [
                    {
                        type: 'text',
                        icon: 'icon-text',
                    },
                    {
                        type: 'picture',
                        icon: 'icon-photo',
                    },
                    {
                        type: 'graphic',
                        icon: 'icon-graphic',
                    },
                    {
                        type: 'composite',
                        icon: 'icon-composite',
                    },
                    {
                        type: 'video',
                        icon: 'icon-video',
                    },
                    {
                        type: 'audio',
                        icon: 'icon-audio',
                    },
                ];

                if (_.get(privileges, 'privileges.planning')) {
                    $scope.fileTypes.push({
                        type: 'event',
                        icon: 'icon-event',
                    });
                    $scope.fileTypes.push({
                        type: 'planning',
                        icon: 'icon-calendar',
                    });
                }

                $scope.minutes = [0, 1, 2, 3, 4, 5, 8, 10, 15, 30, 45];
                $scope.seconds = [0, 5, 10, 15, 30, 45];
                $scope.hours = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
                    13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24];
                $scope.ingestExpiry = appConfig.ingest_expiry_minutes;

                $scope.statusFilters = [
                    {label: gettext('Active'), id: 'active'},
                    {label: gettext('All'), id: 'all'},
                    {label: gettext('Closed'), id: 'closed'},
                ];

                $scope.activeStatusFilter = $scope.statusFilters[0];
                $scope.filterIngestSources = function(id) {
                    $scope.activeStatusFilter = $scope.statusFilters.find((item) => item.id === id);
                    fetchProviders();
                };

                $scope.search = function(query) {
                    if (query) {
                        $scope.searchPage = 1;
                    }
                    $scope.searchQuery = query;
                    fetchProviders();
                };

                function fetchProviders() {
                    var criteria = criteria || {};

                    criteria.max_results = $location.search().max_results || 200;
                    criteria.page = $scope.searchPage || $location.search().page || 1;
                    criteria.sort = 'name';
                    var andTerms = [];

                    if (!_.isEmpty($scope.searchQuery)) {
                        andTerms.push({
                            name: {
                                $regex: $scope.searchQuery,
                                $options: 'i',
                            },
                        });
                    }
                    if ($scope.activeStatusFilter.id !== 'all') {
                        andTerms.push({is_closed: $scope.activeStatusFilter.id !== 'active'});
                    }

                    if (!_.isEmpty(andTerms)) {
                        criteria.where = JSON.stringify({$and: andTerms});
                    }
                    const searchTerm = $scope.searchQuery;

                    return api.ingestProviders.query(criteria)
                        .then((result) => {
                            if ($scope.searchQuery === searchTerm) {
                                $scope.providers = result;
                                $scope.searchPage = null;
                            }
                        });
                }

                function openProviderModal() {
                    var providerId = $location.search()._id;
                    var provider;

                    if (providerId) {
                        if ($scope.providers && $scope.providers._items) {
                            provider = _.find($scope.providers._items, (item) => item._id === providerId);
                        }

                        if (_.isNil(provider)) {
                            api.ingestProviders.getById(providerId).then((result) => {
                                provider = result;
                            });
                        }

                        if (provider) {
                            $scope.edit(provider);
                        }
                    }
                }

                fetchProviders().then(() => {
                    openProviderModal();
                });

                api('rule_sets').query({max_results: 200})
                    .then((result) => {
                        $scope.rulesets = $filter('sortByName')(result._items);
                    });

                api('routing_schemes').query({max_results: 200})
                    .then((result) => {
                        $scope.routingScheme = $filter('sortByName')(result._items);
                    });

                ingestSources.fetchAllFeedParsersAllowed().then((result) => {
                    $scope.allFeedParsers = result;
                    $scope.feedParsers = angular.copy(result);
                });

                /**
                * Returns the result of evaluation of a given expression. Identifiers enclosed in {}
                * are replaced with the values of corresponding fields.
                *
                * @method evalExpression
                * @param {String} expression
                * @return {Boolean}
                */
                function evalExpression(
                    expression,
                    feedingService,
                    provider, // !!! provider is unused, but has to be present in order for eval to have it in context
                ): boolean {
                    if (!feedingService) {
                        return false;
                    }

                    try {
                        // eslint-disable-next-line no-eval
                        return eval(expression);
                    } catch {
                        return false;
                    }
                }

                /**
                * Returns true if the feeding service configuration field was required
                *
                * @method isConfigFieldRequired
                * @param {Object} field
                * @return {Boolean}
                */
                $scope.isConfigFieldRequired = (
                    field,
                    feedingService = $scope.currentFeedingService,
                    provider = $scope.provider,
                ) => {
                    if (field.required) {
                        return true;
                    }
                    if (field.required_expression) {
                        return evalExpression(field.required_expression, feedingService, provider);
                    }
                    return false;
                };

                /**
                * Returns true if the feeding service configuration field was visible
                *
                * @method isConfigFieldVisible
                * @param {Object} field
                * @return {Boolean}
                */
                $scope.isConfigFieldVisible = (
                    field,
                    feedingService = $scope.currentFeedingService,
                    provider = $scope.provider,
                ) => {
                    if (!field.show_expression) {
                        return true;
                    }
                    return evalExpression(field.show_expression, feedingService, provider);
                };

                /**
                 * Should be skipped when adding a new ingest provider that
                 * needs to be saved before it can be authenticated. (gmail ingest)
                 */
                $scope.shouldSkipConfigTest = () => {
                    if ($scope.currentFeedingService == null) {
                        return false;
                    }

                    const urlRequestFields = $scope.currentFeedingService.fields
                        .filter((field) => field.type === 'url_request');

                    const newItemBeingCreated = $scope?.provider?._id == null;

                    return newItemBeingCreated && urlRequestFields.length > 0;
                };

                /**
                * Returns the configuration field HTML identifier
                *
                * @method getConfigFieldId
                * @param {String} fieldId
                * @return {String}
                */
                $scope.getConfigFieldId = (fieldId) => {
                    if (!$scope.provider.feeding_service) {
                        return null;
                    }
                    return $scope.provider.feeding_service + '-' + fieldId;
                };

                /**
                * Fetches the list of errors for the current feeding service
                *
                * @method fetchSourceErrors
                */
                $scope.fetchSourceErrors = function() {
                    if ($scope.provider && $scope.provider.feeding_service) {
                        return api('io_errors').query({source_type: $scope.provider.feeding_service})
                            .then((result) => {
                                $scope.provider.source_errors = result._items[0].source_errors;
                                $scope.provider.all_errors = result._items[0].all_errors;
                            });
                    }
                };

                /**
                * Removed the given ingest provider
                *
                * @method remove
                * @param {Object} provider
                */
                $scope.remove = function(provider) {
                    modal.confirm(gettext('Are you sure you want to delete Ingest Source?')).then(
                        function removeIngestProviderChannel() {
                            api.ingestProviders.remove(provider)
                                .then(
                                    () => {
                                        notify.success(gettext('Ingest Source deleted.'));
                                    },
                                    (response) => {
                                        if (angular.isDefined(response.data._message)) {
                                            notify.error(response.data._message);
                                        } else {
                                            notify.error(gettext('Error: Unable to delete Ingest Source'));
                                        }
                                    },
                                )
                                .then(fetchProviders);
                        },
                    );
                };

                function initTupleFields() {
                    $scope.fieldAliases = {};
                    $scope.fieldsNotSelected = {};
                    $scope.currentFeedingService = $scope.provider ? _.find($scope.feedingServices,
                        {feeding_service: $scope.provider.feeding_service}) : null;

                    if (!$scope.currentFeedingService) {
                        return;
                    }

                    _.forEach($scope.currentFeedingService.fields, (field) => {
                        if (field.default_value !== undefined && $scope.provider.config[field.id] === undefined) {
                            $scope.provider.config[field.id] = field.default_value;
                        }

                        if (field.type === 'mapping') {
                            let aliases = angular.isDefined($scope.provider.config)
                                && $scope.provider.config[field.id] || [];

                            let aliasObj = {};

                            aliases.forEach((item) => {
                                _.extend(aliasObj, item);
                            });

                            $scope.fieldAliases[field.id] = [];
                            Object.keys(aliasObj).forEach((fieldName) => {
                                $scope.fieldAliases[field.id].push(
                                    {fieldName: fieldName, alias: aliasObj[fieldName]});
                            });

                            $scope.fieldsNotSelected[field.id] = field.first_field_options.values.filter(
                                (fieldName) => !(fieldName in aliasObj),
                            );
                        } else if (
                            // preset value in dropdown
                            field.type === 'choices' &&
                            !(field.id in $scope.provider.config) &&
                            'default' in field
                        ) {
                            $scope.provider.config[field.id] = field.default;
                        }
                    });
                }

                $scope.edit = function(provider) { // also gets called when creating a new provider
                    $scope.provider = cloneDeep(provider || {});

                    $scope.provider.update_schedule = $scope.provider.update_schedule
                        || appConfig.ingest.DEFAULT_SCHEDULE;
                    $scope.provider.idle_time = $scope.provider.idle_time || appConfig.ingest.DEFAULT_IDLE_TIME;

                    initTupleFields();
                };

                $scope.cancel = function() {
                    $scope.provider = null;
                    $scope.error = null;
                };

                $scope.setConfig = function(provider) {
                    $scope.provider.config = provider.config;
                };

                /**
                * Appends a new (empty) item to the list of field aliases.
                *
                * @method addFieldAlias
                */
                $scope.addFieldAlias = function(fieldId) {
                    if (!$scope.fieldAliases[fieldId]) {
                        $scope.fieldAliases[fieldId] = [];
                    }
                    $scope.fieldAliases[fieldId].push({fieldName: null, alias: ''});
                };

                /**
                * Removes a field alias from the list of field aliases at the
                * specified index.
                *
                * @method removeFieldAlias
                * @param {Number} itemIdx - index of the item to remove
                */
                $scope.removeFieldAlias = function(fieldId, itemIdx) {
                    var removed = $scope.fieldAliases[fieldId].splice(itemIdx, 1);

                    if (removed[0].fieldName) {
                        $scope.fieldsNotSelected[fieldId].push(removed[0].fieldName);
                    }
                };

                /**
                * Updates the list of content field names not selected in any
                * of the dropdown menus.
                *
                * @method fieldSelectionChanged
                */
                $scope.fieldSelectionChanged = function(field) {
                    var selectedFields = {};

                    $scope.fieldAliases[field.id].forEach((item) => {
                        if (item.fieldName) {
                            selectedFields[item.fieldName] = true;
                        }
                    });

                    $scope.fieldsNotSelected[field.id] = field.first_field_options.values.filter(
                        (fieldName) => !(fieldName in selectedFields),
                    );
                };

                /**
                * Calculates a list of content field names that can be used as
                * options in a dropdown menu.
                *
                * The list is comprised of all field names that are currently
                * not selected in any of the other dropdown menus and
                * of a field name that should be selected in the current
                * dropdown menu (if any).
                *
                * @method availableFieldOptions
                * @param {String} [selectedName] - currently selected field
                * @return {String[]} list of field names
                */
                $scope.availableFieldOptions = function(fieldId, selectedName) {
                    if (!(fieldId in $scope.fieldsNotSelected)) {
                        return [];
                    }
                    var fieldNames = angular.copy($scope.fieldsNotSelected[fieldId]);

                    // add current field selection, if available
                    if (selectedName) {
                        fieldNames.push(selectedName);
                    }
                    return fieldNames;
                };

                /**
                 * Saves the ingest provider.
                 *
                 * @method save
                 */
                $scope.save = function() {
                    _.forEach($scope.currentFeedingService.fields, (field) => {
                        if (field.type !== 'mapping') {
                            return;
                        }
                        let newAliases = [];

                        $scope.fieldAliases[field.id].forEach((item) => {
                            if (item.fieldName && item.alias) {
                                var newAlias = {};

                                newAlias[item.fieldName] = item.alias;
                                newAliases.push(newAlias);
                            }
                        });

                        if (typeof $scope.provider.config !== 'undefined') {
                            $scope.provider.config[field.id] = newAliases;
                        }
                    });
                    delete $scope.provider.all_errors;
                    delete $scope.provider.source_errors;

                    const originalProvider = $scope.providers._items.find(
                        (provider) => provider._id === $scope.provider._id,
                    );

                    $scope.loading = true;

                    const isItemBeingCreated: boolean = originalProvider == null;
                    const urlRequestFields = $scope.currentFeedingService.fields
                        .filter((field) => field.type === 'url_request');

                    if (isItemBeingCreated && urlRequestFields.length > 0) {
                        // See `shouldSkipConfigTest`
                        $scope.provider.skip_config_test = true;
                    }

                    api.ingestProviders.save(originalProvider || {}, $scope.provider)
                        .then((provider) => {
                            const authActions = urlRequestFields
                                .filter((field) => $scope.isConfigFieldVisible(
                                    field,
                                    $scope.currentFeedingService,
                                    provider,
                                ))
                                .map((field) => ({
                                    label: field.label,
                                    onClick: () => {
                                        $scope.doUrlRequest(provider, field);
                                    },
                                }));

                            notify.success(gettext('Provider saved!'));
                            $scope.cancel();
                            $scope.error = null;
                            fetchProviders();

                            if (isItemBeingCreated && urlRequestFields.length > 0 && authActions.length > 0) {
                                authenticateIngestProvider(authActions);
                            }
                        }, (error) => {
                            $scope.error = error.data;
                        })
                        .finally(() => {
                            $scope.loading = null;
                        });
                };

                $scope.gotoIngest = function(provider) {
                    const contentTypes = provider.content_types;
                    const length = provider.content_types.includes('preformatted') || (
                        contentTypes.includes('planning') && contentTypes.includes('event')) ? 2 : 1;

                    if (contentTypes.length === length && (
                        contentTypes.includes('event') || contentTypes.includes('planning'))) {
                        const searchParams = {
                            page: 1,
                            noCalendarAssigned: false,
                            calendars: null,
                            advancedSearch: {
                                source: [{
                                    id: provider._id,
                                    name: provider.name,
                                }],
                            },
                            spikeState: 'draft',
                            fulltext: '',
                        };
                        var filter_value = '';

                        if (contentTypes.includes('event')) {
                            filter_value = 'EVENTS';
                        }
                        if (contentTypes.includes('planning')) {
                            filter_value = 'PLANNING';
                        }
                        if (contentTypes.includes('planning') && contentTypes.includes('event')) {
                            filter_value = 'COMBINED';
                        }
                        $location.path('/planning').search({
                            filter: filter_value,
                            isNewSearch: true,
                            searchParams: angular.toJson(searchParams),
                            calendar: 'ALL_CALENDARS',
                        });
                    } else {
                        $location.path('/search').search(
                            {repo: 'ingest', ingest_provider: provider._id},
                        );
                    }
                };

                /**
                 * Add or remove the current 'fileType' from the provider.
                 *
                 * @param {string} fileType
                 */
                $scope.addOrRemoveFileType = function(fileType, editForm) {
                    editForm.$setDirty();
                    if (!$scope.provider.content_types) {
                        $scope.provider.content_types = [];
                    }

                    var index = $scope.provider.content_types.indexOf(fileType);

                    if (index > -1) {
                        $scope.provider.content_types.splice(index, 1);
                    } else {
                        $scope.provider.content_types.push(fileType);
                    }
                };

                /**
                 * Return true if the 'fileType' is in provider.content_types list.
                 *
                 * @param {string} fileType
                 * @return boolean
                 */
                $scope.hasFileType = function(fileType) {
                    return $scope.provider && $scope.provider.content_types &&
                        $scope.provider.content_types.indexOf(fileType) > -1;
                };

                /**
                 * Initializes the configuration for the selected feeding service.
                 */
                $scope.initProviderConfig = function() {
                    var service: any = getCurrentService();

                    if (service && service.config) {
                        $scope.provider.config = angular.extend({}, service.config);
                    } else {
                        $scope.provider.config = {};
                    }

                    initTupleFields();

                    $scope.provider.feed_parser = null;
                    $scope.feedParsers = angular.copy($scope.allFeedParsers);

                    if ($scope.currentFeedingService) {
                        if ($scope.currentFeedingService.parser_restricted_values &&
                            $scope.currentFeedingService.parser_restricted_values.length) {
                            if ($scope.currentFeedingService.parser_restricted_values.length === 1) {
                                $scope.provider.feed_parser =
                                    $scope.currentFeedingService.parser_restricted_values[0];
                            }
                            $scope.feedParsers = _.filter($scope.feedParsers, (feedParser) =>
                                $scope.currentFeedingService.parser_restricted_values.includes(
                                    feedParser.feed_parser),
                            );
                        } else if ($scope.currentFeedingService.parser_restricted_values === null) {
                            $scope.feedParsers = [];
                        }
                    }
                };

                /**
                 * Returns the HTML src from the templateURL (defined in superdesk-config.js
                 * for the selected feeding service.
                 * @returns {string}
                 */
                $scope.getConfigTemplateUrl = function() {
                    var feedingService: any = getCurrentService();

                    return feedingService ? feedingService.templateUrl : '';
                };

                /**
                 * Do URL request specified in url_request field.
                 * Used for gmail log-in / log-out.
                 * @param provider ingest provider metadata
                 * @param field url_request field metadata
                 */
                $scope.doUrlRequest = (
                    provider: IProvider,
                    field: IFeedingServiceField,
                    hasUnsavedChanges: boolean = false,
                ): void => {
                    if (hasUnsavedChanges) {
                        modal.alert({
                            headerText: gettext('Unsaved changes'),
                            bodyText: gettext('Save all other changes before performing this action.'),
                        });
                    } else {
                        window.open(field.url.replace('{PROVIDER_ID}', provider._id));

                        setTimeout(() => {
                            /**
                             * If websocket message is received indicating a change,
                             * items that are in edit mode are not updated
                             * in order not to lose unsaved data.
                             *
                             * Because it's already checked that there are no unsaved changes,
                             * editing modal is closed now, so after `doUrlRequest` changes the item on the back-end,
                             * front-end will update the provider.
                             *
                             * Otherwise `_etag` wouldn't be updated and it wouldn't work to edit the item.
                             */
                            $scope.cancel();
                        });
                    }
                };

                function getCurrentService() {
                    return _.find($scope.feedingServices, {feeding_service: $scope.provider.feeding_service});
                }

                $scope.$on('$locationChangeSuccess', fetchProviders);

                const eventListenersToRemoveBeforeUnmounting = [];
                const fetchProvidersThrottled = throttle(fetchProviders, 1000);

                eventListenersToRemoveBeforeUnmounting.push(
                    addWebsocketEventListener('resource:updated', (event) => {
                        const {resource, fields} = event.extra;
                        const itemBeingEditedOrCreated = $scope.provider != null && $scope.provider !== false;

                        if (
                            resource === 'ingest_providers'
                            && Object.keys(fields).length > 0
                            && !itemBeingEditedOrCreated
                        ) {
                            fetchProvidersThrottled();
                        }
                    }),
                );

                eventListenersToRemoveBeforeUnmounting.push(
                    addWebsocketEventListener('resource:created', (event) => {
                        const {resource} = event.extra;
                        const itemBeingEditedOrCreated = $scope.provider != null && $scope.provider !== false;

                        if (
                            resource === 'ingest_providers'
                            && !itemBeingEditedOrCreated
                        ) {
                            fetchProvidersThrottled();
                        }
                    }),
                );

                eventListenersToRemoveBeforeUnmounting.push(
                    addWebsocketEventListener('resource:deleted', (event) => {
                        const {resource} = event.extra;
                        const itemBeingEditedOrCreated = $scope.provider != null && $scope.provider !== false;

                        if (
                            resource === 'ingest_providers'
                            && !itemBeingEditedOrCreated
                        ) {
                            fetchProvidersThrottled();
                        }
                    }),
                );

                /**
                 * Display an error message when gmail ingest fails to authenticate
                 */
                function windowMessageHandler(event) {
                    const error = event?.data?.data?.error;

                    if (error != null) {
                        notify.error(error, 'manual');
                    }
                }

                window.addEventListener('message', windowMessageHandler);

                $scope.$on('$destroy', () => {
                    window.removeEventListener('message', windowMessageHandler);

                    eventListenersToRemoveBeforeUnmounting.forEach((removeListener) => {
                        removeListener();
                    });
                });
            });
        },
    };
}