CultureQuestORG/SDP2023

View on GitHub
app/src/main/java/ch/epfl/culturequest/backend/artprocessing/apis/GeneralDescriptionApi.java

Summary

Maintainability
A
3 hrs
Test Coverage
B
89%
package ch.epfl.culturequest.backend.artprocessing.apis;

import com.theokanning.openai.service.OpenAiService;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.function.Function;

import ch.epfl.culturequest.BuildConfig;
import ch.epfl.culturequest.backend.artprocessing.processingobjects.ArtRecognition;
import ch.epfl.culturequest.backend.artprocessing.processingobjects.BasicArtDescription;
import ch.epfl.culturequest.backend.exceptions.OpenAiFailedException;
import ch.epfl.culturequest.backend.exceptions.RecognitionFailedException;
import ch.epfl.culturequest.database.Database;

/**
 * This class combines the results of the WikipediaDescriptionApi and the OpenAIDescriptionApi to get a complete description of the scanned art
 * If the art type is painting or sculpture, the WikipediaDescriptionApi is enough
 * If the art type is architecture or monument, the OpenAIDescriptionApi is used to get the missing data (artist, year, city, country)
 */

public class GeneralDescriptionApi {

    private WikipediaDescriptionApi wikipediaDescriptionApi;
    private OpenAiService service;

    private static final int DEFAULT_SCORE = 50;

    public GeneralDescriptionApi(WikipediaDescriptionApi wikipediaDescriptionApi, OpenAiService openAiService) {
        this.wikipediaDescriptionApi = wikipediaDescriptionApi;
        this.service = openAiService;
    }

    private BasicArtDescription savedWikipediaDescription;

    enum RecoveryState {
        PartialRecovery,
        FullRecovery,
    }

    public CompletableFuture<BasicArtDescription> getArtDescription(ArtRecognition recognizedArt) {

        return wikipediaDescriptionApi.getArtDescription(recognizedArt)
                .thenCompose(basicArtDescription -> {
                        savedWikipediaDescription = basicArtDescription; // We save it in case of subsequent Open AI failure
                        return recoverPotentialMissesAndGetScore(basicArtDescription, recognizedArt, RecoveryState.PartialRecovery, 1); // Wikipedia didn't fail so it either provided a full description or an incomplete one
                    })
                .handle((filledArtDescription, e) -> {
                    if (e != null) {  // Wikipedia or Open AI error

                       // If Open Ai failed and we are in a partial recovery state (since Wikipedia didn't fail), we attempt a second partial recovery with Open AI
                       if(openAiRecoverableFail(e) && savedWikipediaDescription != null){
                           return recoverPotentialMissesAndGetScore(savedWikipediaDescription, recognizedArt, RecoveryState.PartialRecovery, 2);
                       }

                       // if Wikipedia completely fails, we attempt a full recover with OpenAi
                       BasicArtDescription emptyArtDescription = new BasicArtDescription();
                       initializeDescriptionByRecognition(emptyArtDescription, recognizedArt); // initialize the description with the data we already have
                       return recoverPotentialMissesAndGetScore(emptyArtDescription, recognizedArt, RecoveryState.FullRecovery, 2);
                    }

                    Database.setArtwork(filledArtDescription);
                    return CompletableFuture.completedFuture(filledArtDescription); // No initial missing data or Open AI successfully provided the score and recovered the potential missing data
                }).thenCompose(Function.identity());
    }

    private CompletableFuture<BasicArtDescription> recoverPotentialMissesAndGetScore(BasicArtDescription incompleteDescription, ArtRecognition recognizedArt, RecoveryState recoveryState, int tryCount) {

        OpenAIDescriptionApi openAIDescriptionApi = new OpenAIDescriptionApi(service);
        CompletableFuture<Integer> score = openAIDescriptionApi.getScore(recognizedArt);

        ArrayList<String> nullFields = getNullFields(incompleteDescription, recoveryState);

        if (nullFields.size() == 0) { // No missing data, we just have to provide the score. Even it fails here, it doesn't really matter. We just set the score to the default value and still return the object.
            return score
                    .thenApply(s -> {
                        incompleteDescription.setScore(s);
                        return incompleteDescription;
                    })
                    .exceptionally(e -> {
                        incompleteDescription.setScore(DEFAULT_SCORE);
                        Database.setArtwork(incompleteDescription);
                        return incompleteDescription;
                    });
        }


        CompletableFuture<Map<String, String>> missingData = openAIDescriptionApi.getMissingData(recognizedArt, nullFields);

        return missingData.
                thenCombine(score, (data, s) -> {

                    // fill the incomplete basicArtDescription with the missing data & score

                    for (String key : data.keySet()) {
                        try {
                            Field field = BasicArtDescription.class.getDeclaredField(key);
                            field.setAccessible(true);

                            field.set(incompleteDescription, data.get(key));
                        } catch (NoSuchFieldException | IllegalAccessException e) {
                            e.printStackTrace();
                        }
                    }

                    // If Open AI successfully helped for the recovery, we indicate than Open AI was used
                    incompleteDescription.setRequiredOpenAi(true);

                    incompleteDescription.setScore(s);

                    return incompleteDescription;

                })
                .exceptionally(e -> {
                    if (tryCount == 2) {

                        // if both Wikipedia fails and OpenAi fails or Open AI failed two times in a row, we throw an exception leading to a displayed error message in the app
                        throw new CompletionException(new OpenAiFailedException("Open AI critical fail")); // forward the exception to Processing API, it won't get handled by the handle method of getArtDescription method
                    }
                    else {  // Partial recovery

                        throw new CompletionException(new RecognitionFailedException("Open AI recoverable fail")); // forward the exception to handle of getArtDescription method
                    }
                });
    }

    // know which fields are null, the ones Wikipedia didn't find (ambiguity, page doesn't exist, failure, etc.)
    // triggered in partial recovery case
    private ArrayList<String> getNullFields(BasicArtDescription basicArtDescription, RecoveryState recoveryState) {

        ArrayList<String> missingFields = new ArrayList<>();

        // Get all declared fields (including private ones)
        Field[] fields = BasicArtDescription.class.getDeclaredFields();

        // Iterate over the fields
        for (Field field : fields) {

            // Make the field accessible (to access private fields)
            field.setAccessible(true);

            // Get the name and value of the field
            String fieldName = field.getName();
            Object fieldValue;

            if (isIrrelevantFieldForRecovery(fieldName, basicArtDescription.getType())) {
                continue;
            }

            if (recoveryState == RecoveryState.PartialRecovery) {
                try {
                    fieldValue = field.get(basicArtDescription);
                    // if fieldValue is null, add the field name to the list of missing data
                    if (fieldValue == null) {
                        missingFields.add(fieldName);
                    }

                } catch (IllegalAccessException e) {
                    e.printStackTrace();
                }
            }

            // recoveryState == RecoveryState.FullRecovery
            else {
                missingFields.add(fieldName);
            }
        }

        return missingFields;
    }

    // Fields that are not relevant to the OpenAI prompt for missing data
    private Boolean isIrrelevantFieldForRecovery(String fieldName, BasicArtDescription.ArtType artType){

        // if art type is of type architecture, the museum is irrelevant
        if (artType == BasicArtDescription.ArtType.ARCHITECTURE && fieldName == "museum") {
            return true;
        }

        return fieldName == "score" || fieldName == "requiredOpenAi" || fieldName == "type" || fieldName == "name";
    }

    private void initializeDescriptionByRecognition(BasicArtDescription basicArtDescription, ArtRecognition recognizedArt) {
        basicArtDescription.setName(recognizedArt.getArtName());
        basicArtDescription.setType(WikipediaDescriptionApi.getArtType(recognizedArt));
    }

    private boolean openAiRecoverableFail(Throwable exception){
        return exception.getCause() instanceof OpenAiFailedException && exception.getMessage().contains("Open AI recoverable fail");
    }

}