Skip to main content

TestNG Automated App Testing

See our TestNG example repository for a simple example on how to run TestNG mobile app tests.

Installation

First, make sure you install the necessary dependencies to run a test. For this example, we'll be using TestNG and the Appium Java Client.

Add the following dependencies to your pom.xml: 

<dependencies>
    <!-- TestNG -->
    <dependency>
        <groupId>org.testng</groupId>
        <artifactId>testng</artifactId>
        <version>7.11.0</version>
        <scope>test</scope>
    </dependency>
    <!-- Appium Java Client -->
    <dependency>
        <groupId>io.appium</groupId>
        <artifactId>java-client</artifactId>
        <version>10.0.0</version>
        <scope>test</scope>
    </dependency>
</dependencies>

Note: Appium Java Client 10.x requires Java 11 or higher.

TestNG Example

The example below shows a complete TestNG test that connects to the TestingBot hub and runs a test on a real device. In this test, we'll test a sample calculator app by entering values into two input fields and verifying the sum. 

import io.appium.java_client.AppiumBy;
import io.appium.java_client.android.AndroidDriver;
import io.appium.java_client.android.options.UiAutomator2Options;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.*;

import java.net.MalformedURLException;
import java.net.URL;
import java.time.Duration;
import java.util.Map;

public class AndroidAppTest {
    private AndroidDriver driver;

    @BeforeMethod
    public void setUp() throws MalformedURLException {
        // Configure capabilities using UiAutomator2Options
        UiAutomator2Options options = new UiAutomator2Options()
            .setPlatformName("Android")
            .setPlatformVersion("16.0")
            .setDeviceName("Pixel 9")
            .setApp("tb://<your-app-id>")
            .setAutomationName("UiAutomator2");

        // Add TestingBot-specific options
        options.setCapability("tb:options", Map.of(
            "key", "YOUR_TB_KEY",
            "secret", "YOUR_TB_SECRET",
            "name", "Android Calculator Test",
            "build", "TestNG Build 1"
        ));

        // Connect to TestingBot hub
        driver = new AndroidDriver(
            new URL("https://hub.testingbot.com/wd/hub"),
            options
        );
    }

    @Test
    public void testCalculatorSum() {
        WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));

        // Enter first number
        WebElement inputA = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
        );
        inputA.sendKeys("10");

        // Enter second number
        WebElement inputB = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
        );
        inputB.sendKeys("5");

        // Verify the sum
        WebElement sum = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("sum"))
        );
        Assert.assertEquals(sum.getText(), "15");
    }

    @AfterMethod
    public void tearDown() {
        if (driver != null) {
            driver.quit();
        }
    }
}
import io.appium.java_client.AppiumBy;
import io.appium.java_client.ios.IOSDriver;
import io.appium.java_client.ios.options.XCUITestOptions;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.*;

import java.net.MalformedURLException;
import java.net.URL;
import java.time.Duration;
import java.util.Map;

public class IOSAppTest {
    private IOSDriver driver;

    @BeforeMethod
    public void setUp() throws MalformedURLException {
        // Configure capabilities using XCUITestOptions
        XCUITestOptions options = new XCUITestOptions()
            .setPlatformName("iOS")
            .setPlatformVersion("26.0")
            .setDeviceName("iPhone 17")
            .setApp("tb://<your-app-id>")
            .setAutomationName("XCUITest");

        // Add TestingBot-specific options
        options.setCapability("tb:options", Map.of(
            "key", "YOUR_TB_KEY",
            "secret", "YOUR_TB_SECRET",
            "name", "iOS Calculator Test",
            "build", "TestNG Build 1"
        ));

        // Connect to TestingBot hub
        driver = new IOSDriver(
            new URL("https://hub.testingbot.com/wd/hub"),
            options
        );
    }

    @Test
    public void testCalculatorSum() {
        WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));

        // Enter first number
        WebElement inputA = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
        );
        inputA.sendKeys("10");

        // Enter second number
        WebElement inputB = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
        );
        inputB.sendKeys("5");

        // Verify the sum
        WebElement sum = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("sum"))
        );
        Assert.assertEquals(sum.getText(), "15");
    }

    @AfterMethod
    public void tearDown() {
        if (driver != null) {
            driver.quit();
        }
    }
}

Always call driver.quit() in your @AfterMethod to properly end the session. Otherwise, the test will continue running until it times out.

Reusable Base Test Class

For cleaner test organization, you can create a reusable base class that handles driver setup and teardown. Tests can then extend this base class and focus only on test logic.

Base Class

import io.appium.java_client.android.AndroidDriver;
import io.appium.java_client.android.options.UiAutomator2Options;
import org.testng.annotations.*;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;

public class TestingBotBaseTest {
    protected AndroidDriver driver;

    @BeforeMethod(alwaysRun = true)
    @Parameters({"platformVersion", "deviceName", "app"})
    public void setUp(
        @Optional("16.0") String platformVersion,
        @Optional("Pixel 9") String deviceName,
        @Optional("tb://<your-app-id>") String app
    ) throws MalformedURLException {
        // Get credentials from environment or use defaults
        String key = System.getenv("TB_KEY");
        String secret = System.getenv("TB_SECRET");

        if (key == null || secret == null) {
            throw new RuntimeException("TB_KEY and TB_SECRET environment variables must be set");
        }

        // Configure capabilities
        UiAutomator2Options options = new UiAutomator2Options()
            .setPlatformName("Android")
            .setPlatformVersion(platformVersion)
            .setDeviceName(deviceName)
            .setApp(app)
            .setAutomationName("UiAutomator2");

        options.setCapability("tb:options", Map.of(
            "key", key,
            "secret", secret,
            "name", this.getClass().getSimpleName()
        ));

        driver = new AndroidDriver(
            new URL("https://hub.testingbot.com/wd/hub"),
            options
        );
    }

    @AfterMethod(alwaysRun = true)
    public void tearDown() {
        if (driver != null) {
            driver.quit();
        }
    }
}

Test Class Using the Base

import io.appium.java_client.AppiumBy;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.Test;

import java.time.Duration;

public class CalculatorTest extends TestingBotBaseTest {

    @Test
    public void testAddition() {
        WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));

        WebElement inputA = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
        );
        inputA.sendKeys("10");

        WebElement inputB = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
        );
        inputB.sendKeys("5");

        WebElement sum = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("sum"))
        );
        Assert.assertEquals(sum.getText(), "15");
    }
}

TestNG XML Configuration

Use a testng.xml file to pass parameters and run tests on multiple devices: 

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
<suite name="Mobile App Tests" parallel="tests" thread-count="2">
    <test name="Android Pixel 9">
        <parameter name="platformVersion" value="16.0"/>
        <parameter name="deviceName" value="Pixel 9"/>
        <parameter name="app" value="tb://your-app-id"/>
        <classes>
            <class name="CalculatorTest"/>
        </classes>
    </test>
    <test name="Android Pixel 8">
        <parameter name="platformVersion" value="14.0"/>
        <parameter name="deviceName" value="Pixel 8"/>
        <parameter name="app" value="tb://your-app-id"/>
        <classes>
            <class name="CalculatorTest"/>
        </classes>
    </test>
</suite>

Uploading Your App

Please see our Upload Mobile App documentation to find out how to upload your app to TestingBot for testing.

Specify Browsers & Devices

To run your tests on TestingBot, you need to configure the device and platform you want to test on. The Appium Java client provides type-safe Options classes for this purpose.

Using Options Classes (Recommended)

For Android testing, use UiAutomator2Options. For iOS testing, use XCUITestOptions: 

UiAutomator2Options options = new UiAutomator2Options()
    .setPlatformName("Android")
    .setPlatformVersion("16.0")
    .setDeviceName("Pixel 9")
    .setApp("tb://<your-app-id>")
    .setAutomationName("UiAutomator2");

// Add TestingBot credentials
options.setCapability("tb:options", Map.of(
    "key", "YOUR_TB_KEY",
    "secret", "YOUR_TB_SECRET",
    "name", "My Test",
    "build", "Build 1"
));

AndroidDriver driver = new AndroidDriver(
    new URL("https://hub.testingbot.com/wd/hub"),
    options
);
XCUITestOptions options = new XCUITestOptions()
    .setPlatformName("iOS")
    .setPlatformVersion("26.0")
    .setDeviceName("iPhone 17")
    .setApp("tb://<your-app-id>")
    .setAutomationName("XCUITest");

// Add TestingBot credentials
options.setCapability("tb:options", Map.of(
    "key", "YOUR_TB_KEY",
    "secret", "YOUR_TB_SECRET",
    "name", "My Test",
    "build", "Build 1"
));

IOSDriver driver = new IOSDriver(
    new URL("https://hub.testingbot.com/wd/hub"),
    options
);

Using BaseOptions (Generic)

For more flexibility or when working with multiple platforms, you can use BaseOptions: 

import io.appium.java_client.remote.options.BaseOptions;

BaseOptions options = new BaseOptions()
    .setPlatformName("Android")
    .setAutomationName("UiAutomator2")
    .amend("appium:deviceName", "Pixel 9")
    .amend("appium:platformVersion", "16.0")
    .amend("appium:app", "tb://<your-app-id>");

options.setCapability("tb:options", Map.of(
    "key", "YOUR_TB_KEY",
    "secret", "YOUR_TB_SECRET"
));

AppiumDriver driver = new AppiumDriver(
    new URL("https://hub.testingbot.com/wd/hub"),
    options
);

Required Capabilities

To specify which device you want to test on, you need to provide:

  • platformName - The mobile OS (Android or iOS)
  • appium:platformVersion - The OS version (e.g., "16.0", "26.0")
  • appium:deviceName - The device name (e.g., "Pixel 9", "iPhone 17")
  • appium:app - The app to test (URL or tb:// reference)
  • appium:automationName - The automation driver ("UiAutomator2" for Android, "XCUITest" for iOS)

Use the device picker below to generate the configuration for your target device:

Testing Internal Websites

We've built TestingBot Tunnel, to provide you with a secure way to run tests against your staged/internal webapps.
Please see our TestingBot Tunnel documentation for more information about this easy to use tunneling solution.

The example below shows how to easily run a TestNG test with our Tunnel:

1. Download our tunnel and start the tunnel: 

java -jar testingbot-tunnel.jar key secret

2. Update your test to connect through the tunnel instead of directly to the TestingBot hub. Change the URL from hub.testingbot.com to localhost:4445: 

// Connect through the tunnel instead of directly to TestingBot
AndroidDriver driver = new AndroidDriver(
    new URL("http://localhost:4445/wd/hub"),
    options
);

Your test traffic will now go securely through the tunnel, allowing you to test apps that access internal resources.

Mark tests as passed/failed

TestingBot cannot automatically determine if your test passed or failed - that depends on your test assertions. To report test results to TestingBot, use our REST API.

First, add the TestingBot Java client to your dependencies: 

<dependency>
    <groupId>com.testingbot</groupId>
    <artifactId>testingbotrest</artifactId>
    <version>1.0.10</version>
</dependency>

Then use it in your test to report the result: 

import com.testingbot.testingbotrest.TestingbotREST;
import java.util.Map;

@AfterMethod
public void tearDown() {
    if (driver != null) {
        // Report test result to TestingBot
        TestingbotREST api = new TestingbotREST(
            "YOUR_TB_KEY",
            "YOUR_TB_SECRET"
        );

        api.updateTest(
            driver.getSessionId().toString(),
            Map.of(
                "success", "1",  // "1" for passed, "0" for failed
                "name", "Calculator Test"
            )
        );

        driver.quit();
    }
}

For automatic result reporting, you can use a TestNG listener: 

import org.testng.ITestListener;
import org.testng.ITestResult;
import com.testingbot.testingbotrest.TestingbotREST;
import java.util.Map;

public class TestingBotListener implements ITestListener {
    private TestingbotREST api;

    public TestingBotListener() {
        this.api = new TestingbotREST(
            System.getenv("TB_KEY"),
            System.getenv("TB_SECRET")
        );
    }

    @Override
    public void onTestSuccess(ITestResult result) {
        updateTestStatus(result, "1");
    }

    @Override
    public void onTestFailure(ITestResult result) {
        updateTestStatus(result, "0");
    }

    private void updateTestStatus(ITestResult result, String success) {
        Object testInstance = result.getInstance();
        if (testInstance instanceof TestingBotBaseTest) {
            AndroidDriver driver = ((TestingBotBaseTest) testInstance).driver;
            if (driver != null) {
                api.updateTest(
                    driver.getSessionId().toString(),
                    Map.of(
                        "success", success,
                        "name", result.getMethod().getMethodName()
                    )
                );
            }
        }
    }
}

Register the listener in your testng.xml: 

<suite name="Mobile Tests">
    <listeners>
        <listener class-name="TestingBotListener"/>
    </listeners>
    <!-- tests... -->
</suite>

Preparing your App

You do not need to install any code or plugin to run a test.
Below are some things that are necessary to successfully run a mobile test:

For Android:
  • Please supply the URL to your .apk or .aab file.
    Important: the .apk file needs to be a x86 build (x86 ABI) for Android emulators.
For iOS Real Device:
  • Please supply the URL to an .ipa file.
For iOS Simulator:
  • Please supply the URL to a .zip file that contains your .app
  • The .app needs to be an iOS Simulator build:
    • Create a Simulator build:
      xcodebuild -sdk iphonesimulator -configuration Debug
    • Zip the .app bundle:
      zip -r MyApp.zip MyApp.app

Additional Options

Appium provides a lot of options to configure your test.

Some important options that might help:

For Android:
  • appActivity and appPackage: by default, Appium will try to extract the main Activity from your apk. If this fails, you can supply your own with these options.
  • chromeOptions: additional chromedriver options you can supply.
  • otherApps: a JSON array of other apps that need to be installed, in URL format.
For Android & iOS:
  • locale: the language of the simulator/device (ex: fr_CA)

    This sets the locale on the entire device/simulator, except on physical iOS devices. For real iOS devices, we will pass -AppleLocale as argument to the app.

  • newCommandTimeout: How long (in seconds) Appium will wait for a new command from the client before assuming the client quit and ending the session
Was this page helpful?
Last updated
TestingBot Docs