google/EarlGrey

View on GitHub
EarlGrey/Core/GREYKeyboard.m

Summary

Maintainability
Test Coverage
//
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

#import "Core/GREYKeyboard.h"

#include <objc/runtime.h>
#include <stdatomic.h>

#import "Action/GREYTapAction.h"
#import "Additions/NSError+GREYAdditions.h"
#import "Assertion/GREYAssertionDefines.h"
#import "Common/GREYAppleInternals.h"
#import "Common/GREYDefines.h"
#import "Common/GREYError.h"
#import "Common/GREYFatalAsserts.h"
#import "Common/GREYLogger.h"
#import "Core/GREYInteraction.h"
#import "Synchronization/GREYAppStateTracker.h"
#import "Synchronization/GREYAppStateTrackerObject.h"
#import "Synchronization/GREYCondition.h"
#import "Synchronization/GREYUIThreadExecutor.h"
#import "Synchronization/GREYRunLoopSpinner.h"

/**
 *  Action for tapping a keyboard key.
 */
static GREYTapAction *gTapKeyAction;

/**
 *  Flag set to @c true when the keyboard is shown, @c false when keyboard is hidden.
 */
static atomic_bool gIsKeyboardShown = false;

/**
 *  A character set for all alphabets present on a keyboard.
 */
static NSMutableCharacterSet *gAlphabeticKeyplaneCharacters;

/**
 *  Possible accessibility label values for the Shift Key.
 */
static NSArray *gShiftKeyLabels;

/**
 *  Accessibility labels for text modification keys, like shift, delete etc.
 */
static NSDictionary *gModifierKeyIdentifierMapping;

/**
 *  A retry time interval in which we re-tap the shift key to ensure
 *  the alphabetic keyplane changed.
 */
static const NSTimeInterval kMaxShiftKeyToggleDuration = 3.0;

/**
 * Time to wait for the keyboard to appear or disappear.
 */
static const NSTimeInterval kKeyboardWillAppearOrDisappearTimeout = 10.0;

/**
 * Time to spin for the keyboard to change layout.
 */
static const NSTimeInterval kKeyboardLayoutChangeTimeout = 0.1;

/**
 *  Identifier for characters that signify a space key.
 */
static NSString *const kSpaceKeyIdentifier = @" ";

/**
 *  Identifier for characters that signify a delete key.
 */
static NSString *const kDeleteKeyIdentifier = @"\b";

/**
 *  Identifier for characters that signify a return key.
 */
static NSString *const kReturnKeyIdentifier = @"\n";

@implementation GREYKeyboard : NSObject

+ (void)load {
  @autoreleasepool {
    NSObject *keyboardObject = [[NSObject alloc] init];
    // Hooks to keyboard lifecycle notification.
    NSNotificationCenter *defaultNotificationCenter = [NSNotificationCenter defaultCenter];
    [defaultNotificationCenter addObserverForName:UIKeyboardWillShowNotification
                                           object:nil
                                            queue:nil
                                       usingBlock:^(NSNotification *note) {
      GREYAppStateTrackerObject *object =
          TRACK_STATE_FOR_OBJECT(kGREYPendingKeyboardTransition, keyboardObject);
      objc_setAssociatedObject(keyboardObject,
                               @selector(grey_keyboardObject),
                               object,
                               OBJC_ASSOCIATION_RETAIN_NONATOMIC);
    }];
    [defaultNotificationCenter addObserverForName:UIKeyboardDidShowNotification
                                           object:nil
                                            queue:nil
                                       usingBlock:^(NSNotification *note) {
      GREYAppStateTrackerObject *object =
          objc_getAssociatedObject(keyboardObject, @selector(grey_keyboardObject));
      UNTRACK_STATE_FOR_OBJECT(kGREYPendingKeyboardTransition, object);
      // There may be a zero size inputAccessoryView to track keyboard data.
      // This causes UIKeyboardDidShowNotification event to fire even though no keyboard is visible.
      // So intead of relying on keyboard show/hide event to detect the keyboard visibility, it is
      // necessary to double check on the actual frame to determine the true visibility.
      CGRect keyboardFrame = [note.userInfo[UIKeyboardFrameEndUserInfoKey] CGRectValue];
      UIWindow *window = [UIApplication sharedApplication].keyWindow;
      keyboardFrame = [window convertRect:keyboardFrame fromWindow:nil];
      CGRect windowFrame = window.frame;
      CGRect frameIntersection = CGRectIntersection(windowFrame, keyboardFrame);
      bool keyboardVisible = frameIntersection.size.width > 1 && frameIntersection.size.height > 1;
      atomic_store(&gIsKeyboardShown, keyboardVisible);
    }];
    [defaultNotificationCenter addObserverForName:UIKeyboardWillHideNotification
                                           object:nil
                                            queue:nil
                                       usingBlock:^(NSNotification *note) {
      atomic_store(&gIsKeyboardShown, false);
      GREYAppStateTrackerObject *object =
          TRACK_STATE_FOR_OBJECT(kGREYPendingKeyboardTransition, keyboardObject);
      objc_setAssociatedObject(keyboardObject,
                               @selector(grey_keyboardObject),
                               object,
                               OBJC_ASSOCIATION_RETAIN_NONATOMIC);
    }];
    [defaultNotificationCenter addObserverForName:UIKeyboardDidHideNotification
                                           object:nil
                                            queue:nil
                                       usingBlock:^(NSNotification *note) {
      GREYAppStateTrackerObject *object =
          objc_getAssociatedObject(keyboardObject, @selector(grey_keyboardObject));
      UNTRACK_STATE_FOR_OBJECT(kGREYPendingKeyboardTransition, object);
    }];
  }
}

+ (void)initialize {
  if (self == [GREYKeyboard class]) {
    gTapKeyAction = [[GREYTapAction alloc] initWithType:kGREYTapTypeKBKey];
    // Note: more, numbers label must be after shift and SHIFT labels, because it is also used for
    // the key for switching between keyplanes.
    gShiftKeyLabels =
        @[ @"shift", @"Shift", @"SHIFT", @"more, symbols", @"more, numbers", @"numbers", @"more", @"MORE" ];

    NSCharacterSet *lowerCaseSet = [NSCharacterSet lowercaseLetterCharacterSet];
    gAlphabeticKeyplaneCharacters = [NSMutableCharacterSet uppercaseLetterCharacterSet];
    [gAlphabeticKeyplaneCharacters formUnionWithCharacterSet:lowerCaseSet];

    gModifierKeyIdentifierMapping = @{
        kSpaceKeyIdentifier : @"space",
        kDeleteKeyIdentifier : @"delete",
        kReturnKeyIdentifier : @"return"
    };
  }
}

+ (BOOL)typeString:(NSString *)string
    inFirstResponder:(id)firstResponder
               error:(__strong NSError **)errorOrNil {
  if ([string length] < 1) {
    GREYPopulateErrorOrLog(errorOrNil,
                           kGREYInteractionErrorDomain,
                           kGREYInteractionActionFailedErrorCode,
                           @"Failed to type, because the string provided was empty.");

    return NO;
  } else if (!atomic_load(&gIsKeyboardShown)) {
    NSString *description = [NSString stringWithFormat:@"Failed to type string '%@', "
                                                       @"because keyboard was not shown on screen.",
                             string];

    GREYPopulateErrorOrLog(errorOrNil,
                           kGREYInteractionErrorDomain,
                           kGREYInteractionActionFailedErrorCode,
                           description);

    return NO;
  }

  __block BOOL success = YES;
  for (NSUInteger i = 0; ((i < string.length) && success); i++) {
    NSString *characterAsString = [NSString stringWithFormat:@"%C", [string characterAtIndex:i]];
    NSLog(@"Attempting to type key %@.", characterAsString);

    id key = [GREYKeyboard grey_waitAndfindKeyForCharacter:characterAsString];
    // If key is not on the screen, try looking for it on another keyplane.
    if (!key) {
      unichar currentCharacter = [characterAsString characterAtIndex:0];
      if ([gAlphabeticKeyplaneCharacters characterIsMember:currentCharacter]) {
        GREYLogVerbose(@"Detected an alphabetic key.");
        // Switch to alphabetic keyplane if we are on numbers/symbols keyplane.
        if (![GREYKeyboard grey_isAlphabeticKeyplaneShown]) {
          NSString *moreLettersKeyAxLabel = iOS13_OR_ABOVE() ? @"letters" : @"more, letters";
          id moreLettersKey = [GREYKeyboard grey_waitAndfindKeyForCharacter:moreLettersKeyAxLabel];
          if (!moreLettersKey) {
            return [GREYKeyboard grey_setErrorForkeyNotFoundWithAccessibilityLabel:moreLettersKeyAxLabel
                                                                   forTypingString:string
                                                                             error:errorOrNil];
          }
          [GREYKeyboard grey_tapKey:moreLettersKey error:errorOrNil];
          key = [GREYKeyboard grey_waitAndfindKeyForCharacter:characterAsString];
        }
        // If key is not on the current keyplane, use shift to switch to the other one.
        if (!key) {
          key = [GREYKeyboard grey_toggleShiftAndFindKeyWithAccessibilityLabel:characterAsString
                                                                     withError:errorOrNil];
        }
      } else {
        GREYLogVerbose(@"Detected a non-alphabetic key.");
        // Switch to numbers/symbols keyplane if we are on alphabetic keyplane.
        if ([GREYKeyboard grey_isAlphabeticKeyplaneShown]) {
          NSString *moreNumberKeyAxLabel = iOS13_OR_ABOVE() ? @"numbers" : @"more, numbers";
          id moreNumbersKey = [GREYKeyboard grey_waitAndfindKeyForCharacter:moreNumberKeyAxLabel];
          if (!moreNumbersKey) {
            return
                [GREYKeyboard grey_setErrorForkeyNotFoundWithAccessibilityLabel:moreNumberKeyAxLabel
                                                                forTypingString:string
                                                                          error:errorOrNil];
          }
          [GREYKeyboard grey_tapKey:moreNumbersKey error:errorOrNil];
          key = [GREYKeyboard grey_waitAndfindKeyForCharacter:characterAsString];
        }
        // If key is not on the current keyplane, use shift to switch to the other one.
        if (!key) {
          if (![GREYKeyboard grey_toggleShiftKeyWithError:errorOrNil]) {
            success = NO;
            break;
          }
          key = [GREYKeyboard grey_waitAndfindKeyForCharacter:characterAsString];
        }
        // If key is not on either number or symbols keyplane, it could be on alphabetic keyplane.
        // This is the case for @ _ - on UIKeyboardTypeEmailAddress on iPad.
        if (!key) {
          NSString *moreLettersKeyAxLabel = iOS13_OR_ABOVE() ? @"letters" : @"more, letters";
          id moreLettersKey = [GREYKeyboard grey_waitAndfindKeyForCharacter:moreLettersKeyAxLabel];
          if (!moreLettersKey) {
            return [GREYKeyboard grey_setErrorForkeyNotFoundWithAccessibilityLabel:moreLettersKeyAxLabel
                                                                   forTypingString:string
                                                                             error:errorOrNil];
          }
          [GREYKeyboard grey_tapKey:moreLettersKey error:errorOrNil];
          key = [GREYKeyboard grey_waitAndfindKeyForCharacter:characterAsString];
        }
      }
      // If key is still not shown on screen, show error message.
      if (!key) {
        return [GREYKeyboard grey_setErrorForkeyNotFoundWithAccessibilityLabel:characterAsString
                                                               forTypingString:string
                                                                         error:errorOrNil];
      }
    }
    // A period key for an email UITextField on iOS9 and above types the email domain (.com, .org)
    // by default. That is not the desired behavior so check below disables it.
    BOOL keyboardTypeWasChangedFromEmailType = NO;
    if (iOS9_OR_ABOVE() &&
        [characterAsString isEqualToString:@"."] &&
        [firstResponder respondsToSelector:@selector(keyboardType)] &&
        [firstResponder keyboardType] == UIKeyboardTypeEmailAddress) {
      [firstResponder setKeyboardType:UIKeyboardTypeDefault];
      keyboardTypeWasChangedFromEmailType = YES;
    }

    // Keyboard was found; this action should always succeed.
    [GREYKeyboard grey_tapKey:key error:errorOrNil];

    // When space, delete or uppercase letter is typed, the keyboard will automatically change to
    // lower alphabet keyplane.
    // On iPad the layout changes faster than accessibility, so we need to wait for
    // accessibility change.
    unichar character = [characterAsString characterAtIndex:0];
    if ([characterAsString isEqualToString:kSpaceKeyIdentifier] ||
        [characterAsString isEqualToString:kDeleteKeyIdentifier] ||
        [[NSCharacterSet uppercaseLetterCharacterSet] characterIsMember:character]) {
      [GREYKeyboard grey_waitAndfindKeyForCharacter:@"e"];
    }

    if (keyboardTypeWasChangedFromEmailType) {
      // Set the keyboard type back to the Email Type.
      [firstResponder setKeyboardType:UIKeyboardTypeEmailAddress];
    }
  }

  return success;
}

+ (BOOL)waitForKeyboardToAppear {
  if (atomic_load(&gIsKeyboardShown)) {
    return YES;
  }
  GREYCondition *keyboardIsShownCondition =
      [[GREYCondition alloc] initWithName:@"Keyboard will appear." block:^BOOL {
        return atomic_load(&gIsKeyboardShown);
      }];
  return [keyboardIsShownCondition waitWithTimeout:kKeyboardWillAppearOrDisappearTimeout];
}

+ (BOOL)isKeyboardShown {
  return atomic_load(&gIsKeyboardShown);
}

#pragma mark - Private

/**
 *  A utility method to continuously toggle the shift key on an alphabet keyplane until
 *  the correct character case is found.
 *
 *  @param      accessibilityLabel The accessibility label of the key for which
 *                                 the case is being changed.
 *  @param[out] errorOrNil         Error populated on failure.
 *
 *  @return The case toggled key for the accessibility label, or @c nil if it isn't found.
 */
+ (id)grey_toggleShiftAndFindKeyWithAccessibilityLabel:(NSString *)accessibilityLabel
                                           withError:(__strong NSError **)errorOrNil {
  __block id key = nil;
  __block NSError *error;
  GREYCondition *shiftToggleSucceded =
      [GREYCondition conditionWithName:@"Shift key toggled keyplane" block:^BOOL() {
     [GREYKeyboard grey_toggleShiftKeyWithError:&error];
     key = [GREYKeyboard grey_waitAndfindKeyForCharacter:accessibilityLabel];
     return (key != nil) || (error != nil);
   }];

  BOOL didTimeOut = ![shiftToggleSucceded waitWithTimeout:kMaxShiftKeyToggleDuration];
  if (didTimeOut) {
    GREYPopulateErrorOrLog(errorOrNil,
                           kGREYInteractionErrorDomain,
                           kGREYInteractionTimeoutErrorCode,
                           @"GREYKeyboard : Shift Key toggling timed out "
                           @"since key with correct case wasn't found");
  }

  return key;
}

/**
 *  Private API to toggle shift, because tapping on the key was flaky and required a 0.35 second
 *  wait due to accidental touch detection. The 0.35 seconds is the value within which, if a second
 *  tap occurs, then a double tap is registered.
 *
 *  @param[out] errorOrNil Error populated on failure.
 *
 *  @return YES if the shift toggle succeeded, else NO.
 */
+ (BOOL)grey_toggleShiftKeyWithError:(__strong NSError **)errorOrNil {
  GREYLogVerbose(@"Tapping on Shift key.");
  UIKeyboardImpl *keyboard = [GREYKeyboard grey_keyboardObject];
  // Clear time Shift key was pressed last to make sure the keyboard will not ignore this event.
  // If we do not reset this value, we would need to wait at least 0.35 seconds after toggling
  // Shift before we could reliably toggle it again. This is likely related to the double-tap
  // gesture used for shift-lock (also called caps-lock).
  [[keyboard _layout] setValue:[NSNumber numberWithDouble:0.0] forKey:@"_shiftLockFirstTapTime"];

  for (NSString *shiftKeyLabel in gShiftKeyLabels) {
    id key = [GREYKeyboard grey_waitAndfindKeyForCharacter:shiftKeyLabel];
    if (key) {
      // Shift key was found; this action should always succeed.
      [GREYKeyboard grey_tapKey:key error:errorOrNil];
      return YES;
    }
  }
  GREYPopulateErrorOrLog(errorOrNil,
                         kGREYInteractionErrorDomain,
                         kGREYInteractionActionFailedErrorCode,
                         @"GREYKeyboard: No known SHIFT key was found in the hierarchy.");
  return NO;
}

/**
 *  Get the key on the keyboard for a character to be typed. Will wait for the character if it is
 *  not on the keyboard layout yet.
 *
 *  @param character The character that needs to be typed.
 *
 *  @return A UI element that signifies the key to be tapped for typing action.
 */
+ (id)grey_waitAndfindKeyForCharacter:(NSString *)character {
  GREYFatalAssert(character);

  BOOL ignoreCase = NO;
  // If the key is a modifier key then we need to do a case-insensitive comparison and change the
  // accessibility label to the corresponding modifier key accessibility label.
  NSString *modifierKeyIdentifier = [gModifierKeyIdentifierMapping objectForKey:character];
  if (modifierKeyIdentifier) {
    // Check for the return key since we can have a different accessibility label
    // depending upon the keyboard.
    UIKeyboardImpl *currentKeyboard = [GREYKeyboard grey_keyboardObject];
    if ([character isEqualToString:kReturnKeyIdentifier]) {
      modifierKeyIdentifier = [currentKeyboard returnKeyDisplayName];
    }
    character = modifierKeyIdentifier;
    ignoreCase = YES;
  }

  // iOS 9 changes & to ampersand.
  if ([character isEqualToString:@"&"] && iOS9_OR_ABOVE()) {
    character = @"ampersand";
  }

  __block id result = nil;
  [GREYKeyboard grey_spinRunloopForKeyboardWithTimeout:kKeyboardLayoutChangeTimeout
                                  andStoppingCondition:^BOOL {
    result = [self grey_keyForCharacterValue:character
         inKeyboardLayoutWithCaseSensitivity:ignoreCase];
    return result != nil;
  }];
  return result;
}

/**
 *  Get the key on the keyboard for the given accessibility label.
 *
 *  @param character  A string identifying the key to be searched.
 *  @param ignoreCase A Boolean that is @c YES if searching for the key requires ignoring
 *                    the case. This is seen in the case of modifier keys that have
 *                    differing cases across iOS versions.
 *
 *  @return A key that has the given accessibility label.
 */
+ (id)grey_keyForCharacterValue:(NSString *)character
    inKeyboardLayoutWithCaseSensitivity:(BOOL)ignoreCase {
  UIKeyboardImpl *keyboard = [GREYKeyboard grey_keyboardObject];
  // Type of layout is private class UIKeyboardLayoutStar, which implements UIAccessibilityContainer
  // Protocol and contains accessibility elements for keyboard keys that it shows on the screen.
  id layout = [keyboard _layout];
  GREYFatalAssertWithMessage(layout, @"Layout instance must not be nil");
  if ([layout accessibilityElementCount] != NSNotFound) {
    for (NSInteger i = 0; i < [layout accessibilityElementCount]; ++i) {
      id key = [layout accessibilityElementAtIndex:i];
      if ((ignoreCase &&
           [[key accessibilityLabel] caseInsensitiveCompare:character] == NSOrderedSame) ||
          (!ignoreCase && [[key accessibilityLabel] isEqualToString:character])) {
        return key;
      }

      if ([key accessibilityIdentifier] &&
          [[key accessibilityIdentifier] isEqualToString:character]) {
        return key;
      }
    }
  }
  return nil;
}

/**
 *  A flag to check if the alphabetic keyplan is currently visible on the keyboard.
 *
 *  @return @c YES if the alphabetic keyplan is being shown on the keyboard, else @c NO.
 */
+ (BOOL)grey_isAlphabeticKeyplaneShown {
  // Arbitrarily choose e/E as the key to look for to determine if alphabetic keyplane is shown.
  return [GREYKeyboard grey_waitAndfindKeyForCharacter:@"e"] != nil
      || [GREYKeyboard grey_waitAndfindKeyForCharacter:@"E"] != nil;
}

/**
 *  Provides the active keyboard instance.
 *
 *  @return The active UIKeyboardImpl instance.
 */
+ (UIKeyboardImpl *)grey_keyboardObject {
  UIKeyboardImpl *keyboard = [UIKeyboardImpl activeInstance];
  GREYFatalAssertWithMessage(keyboard, @"Keyboard instance must not be nil");
  return keyboard;
}

/**
 *  Utility method to tap on a key on the keyboard.
 *
 *  @param      key        The key to be tapped.
 *  @param[out] errorOrNil The error to be populated. If this is @c nil,
 *                         then an error message is logged.
 */
+ (BOOL)grey_tapKey:(id)key error:(__strong NSError **)errorOrNil {
  GREYFatalAssert(key);

  NSLog(@"Tapping on key: %@.", [key accessibilityLabel]);
  BOOL success = [gTapKeyAction perform:key error:errorOrNil];
  [[[GREYKeyboard grey_keyboardObject] taskQueue] waitUntilAllTasksAreFinished];
  [[GREYUIThreadExecutor sharedInstance] drainOnce];
  return success;
}

/**
 *  Populates or prints an error whenever a key with an accessibility label isn't found during
 *  typing a string.
 *
 *  @param accessibilityLabel The accessibility label of the key
 *  @param string             The string being typed when the key was not found
 *  @param[out] errorOrNil    The error to be populated. If this is @c nil,
 *                            then an error message is logged.
 *
 *  @return NO every time since entering the method means an error has happened.
 */
+ (BOOL)grey_setErrorForkeyNotFoundWithAccessibilityLabel:(NSString *)accessibilityLabel
                                          forTypingString:(NSString *)string
                                                    error:(__strong NSError **)errorOrNil {
  NSString *description = [NSString stringWithFormat:@"Failed to type string '%@', "
                                                     @"because key [K] could not be found "
                                                     @"on the keyboard.",
                                                     string];
  NSDictionary *glossary = @{ @"K" : [accessibilityLabel description] };
  GREYPopulateErrorNotedOrLog(errorOrNil,
                              kGREYInteractionErrorDomain,
                              kGREYInteractionElementNotFoundErrorCode,
                              description,
                              glossary);
  return NO;
}

/**
 *  To wait for a keyboard animation or gesture, spin the runloop.
 *
 *  @param timeout            The timeout of the runloop spinner.
 *  @param stopConditionBlock The condition block used to stop the runloop spinner.
 */
+ (void)grey_spinRunloopForKeyboardWithTimeout:(NSTimeInterval)timeout
                          andStoppingCondition:(BOOL (^)(void))stopConditionBlock {
  GREYRunLoopSpinner *runLoopSpinner = [[GREYRunLoopSpinner alloc] init];
  runLoopSpinner.timeout = timeout;
  runLoopSpinner.maxSleepInterval = DBL_MAX;
  [runLoopSpinner spinWithStopConditionBlock:stopConditionBlock];
}

@end