Skip to main content

TestNG Automated App Testing

See our TestNG example repository for a simple example on how to run TestNG mobile app tests.

Installation

First, make sure you install the necessary dependencies to run a test. For this example, we'll be using TestNG and the Appium Java Client.

Add the following dependencies to your pom.xml: 

<dependencies>
    <!-- TestNG -->
    <dependency>
        <groupId>org.testng</groupId>
        <artifactId>testng</artifactId>
        <version>7.11.0</version>
        <scope>test</scope>
    </dependency>
    <!-- Appium Java Client -->
    <dependency>
        <groupId>io.appium</groupId>
        <artifactId>java-client</artifactId>
        <version>10.0.0</version>
        <scope>test</scope>
    </dependency>
</dependencies>

Note: Appium Java Client 10.x requires Java 11 or higher.

TestNG Example

The example below shows a complete TestNG test that connects to the TestingBot hub and runs a test on a real device. In this test, we'll test a sample calculator app by entering values into two input fields and verifying the sum. 

import io.appium.java_client.AppiumBy;
import io.appium.java_client.android.AndroidDriver;
import io.appium.java_client.android.options.UiAutomator2Options;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.*;

import java.net.MalformedURLException;
import java.net.URL;
import java.time.Duration;
import java.util.Map;

public class AndroidAppTest {
    private AndroidDriver driver;

    @BeforeMethod
    public void setUp() throws MalformedURLException {
        // Configure capabilities using UiAutomator2Options
        UiAutomator2Options options = new UiAutomator2Options()
            .setPlatformName("Android")
            .setPlatformVersion("16.0")
            .setDeviceName("Pixel 9")
            .setApp("tb://<your-app-id>")
            .setAutomationName("UiAutomator2");

        // Add TestingBot-specific options
        options.setCapability("tb:options", Map.of(
            "key", "YOUR_TB_KEY",
            "secret", "YOUR_TB_SECRET",
            "name", "Android Calculator Test",
            "build", "TestNG Build 1"
        ));

        // Connect to TestingBot hub
        driver = new AndroidDriver(
            new URL("https://hub.testingbot.com/wd/hub"),
            options
        );
    }

    @Test
    public void testCalculatorSum() {
        WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));

        // Enter first number
        WebElement inputA = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
        );
        inputA.sendKeys("10");

        // Enter second number
        WebElement inputB = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
        );
        inputB.sendKeys("5");

        // Verify the sum
        WebElement sum = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("sum"))
        );
        Assert.assertEquals(sum.getText(), "15");
    }

    @AfterMethod
    public void tearDown() {
        if (driver != null) {
            driver.quit();
        }
    }
}
import io.appium.java_client.AppiumBy;
import io.appium.java_client.ios.IOSDriver;
import io.appium.java_client.ios.options.XCUITestOptions;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.*;

import java.net.MalformedURLException;
import java.net.URL;
import java.time.Duration;
import java.util.Map;

public class IOSAppTest {
    private IOSDriver driver;

    @BeforeMethod
    public void setUp() throws MalformedURLException {
        // Configure capabilities using XCUITestOptions
        XCUITestOptions options = new XCUITestOptions()
            .setPlatformName("iOS")
            .setPlatformVersion("26.0")
            .setDeviceName("iPhone 17")
            .setApp("tb://<your-app-id>")
            .setAutomationName("XCUITest");

        // Add TestingBot-specific options
        options.setCapability("tb:options", Map.of(
            "key", "YOUR_TB_KEY",
            "secret", "YOUR_TB_SECRET",
            "name", "iOS Calculator Test",
            "build", "TestNG Build 1"
        ));

        // Connect to TestingBot hub
        driver = new IOSDriver(
            new URL("https://hub.testingbot.com/wd/hub"),
            options
        );
    }

    @Test
    public void testCalculatorSum() {
        WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));

        // Enter first number
        WebElement inputA = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
        );
        inputA.sendKeys("10");

        // Enter second number
        WebElement inputB = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
        );
        inputB.sendKeys("5");

        // Verify the sum
        WebElement sum = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("sum"))
        );
        Assert.assertEquals(sum.getText(), "15");
    }

    @AfterMethod
    public void tearDown() {
        if (driver != null) {
            driver.quit();
        }
    }
}

Always call driver.quit() in your @AfterMethod to properly end the session. Otherwise, the test will continue running until it times out.

Reusable Base Test Class

For cleaner test organization, you can create a reusable base class that handles driver setup and teardown. Tests can then extend this base class and focus only on test logic.

Base Class

import io.appium.java_client.android.AndroidDriver;
import io.appium.java_client.android.options.UiAutomator2Options;
import org.testng.annotations.*;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;

public class TestingBotBaseTest {
    protected AndroidDriver driver;

    @BeforeMethod(alwaysRun = true)
    @Parameters({"platformVersion", "deviceName", "app"})
    public void setUp(
        @Optional("16.0") String platformVersion,
        @Optional("Pixel 9") String deviceName,
        @Optional("tb://<your-app-id>") String app
    ) throws MalformedURLException {
        // Get credentials from environment or use defaults
        String key = System.getenv("TB_KEY");
        String secret = System.getenv("TB_SECRET");

        if (key == null || secret == null) {
            throw new RuntimeException("TB_KEY and TB_SECRET environment variables must be set");
        }

        // Configure capabilities
        UiAutomator2Options options = new UiAutomator2Options()
            .setPlatformName("Android")
            .setPlatformVersion(platformVersion)
            .setDeviceName(deviceName)
            .setApp(app)
            .setAutomationName("UiAutomator2");

        options.setCapability("tb:options", Map.of(
            "key", key,
            "secret", secret,
            "name", this.getClass().getSimpleName()
        ));

        driver = new AndroidDriver(
            new URL("https://hub.testingbot.com/wd/hub"),
            options
        );
    }

    @AfterMethod(alwaysRun = true)
    public void tearDown() {
        if (driver != null) {
            driver.quit();
        }
    }
}

Test Class Using the Base

import io.appium.java_client.AppiumBy;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.Test;

import java.time.Duration;

public class CalculatorTest extends TestingBotBaseTest {

    @Test
    public void testAddition() {
        WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));

        WebElement inputA = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
        );
        inputA.sendKeys("10");

        WebElement inputB = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
        );
        inputB.sendKeys("5");

        WebElement sum = wait.until(
            ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("sum"))
        );
        Assert.assertEquals(sum.getText(), "15");
    }
}

TestNG XML Configuration

Use a testng.xml file to pass parameters and run tests on multiple devices: 

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
<suite name="Mobile App Tests" parallel="tests" thread-count="2">
    <test name="Android Pixel 9">
        <parameter name="platformVersion" value="16.0"/>
        <parameter name="deviceName" value="Pixel 9"/>
        <parameter name="app" value="tb://your-app-id"/>
        <classes>
            <class name="CalculatorTest"/>
        </classes>
    </test>
    <test name="Android Pixel 8">
        <parameter name="platformVersion" value="14.0"/>
        <parameter name="deviceName" value="Pixel 8"/>
        <parameter name="app" value="tb://your-app-id"/>
        <classes>
            <class name="CalculatorTest"/>
        </classes>
    </test>
</suite>

Uploading Your App

Please see our Upload Mobile App documentation to find out how to upload your app to TestingBot for testing.

Specify Browsers & Devices

To run your tests on TestingBot, you need to configure the device and platform you want to test on. The Appium Java client provides type-safe Options classes for this purpose.

Using Options Classes (Recommended)

For Android testing, use UiAutomator2Options. For iOS testing, use XCUITestOptions: 

UiAutomator2Options options = new UiAutomator2Options()
    .setPlatformName("Android")
    .setPlatformVersion("16.0")
    .setDeviceName("Pixel 9")
    .setApp("tb://<your-app-id>")
    .setAutomationName("UiAutomator2");

// Add TestingBot credentials
options.setCapability("tb:options", Map.of(
    "key", "YOUR_TB_KEY",
    "secret", "YOUR_TB_SECRET",
    "name", "My Test",
    "build", "Build 1"
));

AndroidDriver driver = new AndroidDriver(
    new URL("https://hub.testingbot.com/wd/hub"),
    options
);
XCUITestOptions options = new XCUITestOptions()
    .setPlatformName("iOS")
    .setPlatformVersion("26.0")
    .setDeviceName("iPhone 17")
    .setApp("tb://<your-app-id>")
    .setAutomationName("XCUITest");

// Add TestingBot credentials
options.setCapability("tb:options", Map.of(
    "key", "YOUR_TB_KEY",
    "secret", "YOUR_TB_SECRET",
    "name", "My Test",
    "build", "Build 1"
));

IOSDriver driver = new IOSDriver(
    new URL("https://hub.testingbot.com/wd/hub"),
    options
);

Using BaseOptions (Generic)

For more flexibility or when working with multiple platforms, you can use BaseOptions: 

import io.appium.java_client.remote.options.BaseOptions;

BaseOptions options = new BaseOptions()
    .setPlatformName("Android")
    .setAutomationName("UiAutomator2")
    .amend("appium:deviceName", "Pixel 9")
    .amend("appium:platformVersion", "16.0")
    .amend("appium:app", "tb://<your-app-id>");

options.setCapability("tb:options", Map.of(
    "key", "YOUR_TB_KEY",
    "secret", "YOUR_TB_SECRET"
));

AppiumDriver driver = new AppiumDriver(
    new URL("https://hub.testingbot.com/wd/hub"),
    options
);

Required Capabilities

To specify which device you want to test on, you need to provide:

  • platformName - The mobile OS (Android or iOS)
  • appium:platformVersion - The OS version (e.g., "16.0", "26.0")
  • appium:deviceName - The device name (e.g., "Pixel 9", "iPhone 17")
  • appium:app - The app to test (URL or tb:// reference)
  • appium:automationName - The automation driver ("UiAutomator2" for Android, "XCUITest" for iOS)

Use the device picker below to generate the configuration for your target device:

Testing Internal Websites

We've built TestingBot Tunnel, to provide you with a secure way to run tests against your staged/internal webapps.
Please see our TestingBot Tunnel documentation for more information about this easy to use tunneling solution.

The example below shows how to easily run a TestNG test with our Tunnel:

1. Download our tunnel and start the tunnel: 

java -jar testingbot-tunnel.jar key secret

2. Update your test to connect through the tunnel instead of directly to the TestingBot hub. Change the URL from hub.testingbot.com to localhost:4445: 

// Connect through the tunnel instead of directly to TestingBot
AndroidDriver driver = new AndroidDriver(
    new URL("http://localhost:4445/wd/hub"),
    options
);

Your test traffic will now go securely through the tunnel, allowing you to test apps that access internal resources.

Mark tests as passed/failed

TestingBot cannot automatically determine if your test passed or failed - that depends on your test assertions. To report test results to TestingBot, use our REST API.

First, add the TestingBot Java client to your dependencies: 

<dependency>
    <groupId>com.testingbot</groupId>
    <artifactId>testingbotrest</artifactId>
    <version>1.0.10</version>
</dependency>

Then use it in your test to report the result: 

import com.testingbot.testingbotrest.TestingbotREST;
import java.util.Map;

@AfterMethod
public void tearDown() {
    if (driver != null) {
        // Report test result to TestingBot
        TestingbotREST api = new TestingbotREST(
            "YOUR_TB_KEY",
            "YOUR_TB_SECRET"
        );

        api.updateTest(
            driver.getSessionId().toString(),
            Map.of(
                "success", "1",  // "1" for passed, "0" for failed
                "name", "Calculator Test"
            )
        );

        driver.quit();
    }
}

For automatic result reporting, you can use a TestNG listener: 

import org.testng.ITestListener;
import org.testng.ITestResult;
import com.testingbot.testingbotrest.TestingbotREST;
import java.util.Map;

public class TestingBotListener implements ITestListener {
    private TestingbotREST api;

    public TestingBotListener() {
        this.api = new TestingbotREST(
            System.getenv("TB_KEY"),
            System.getenv("TB_SECRET")
        );
    }

    @Override
    public void onTestSuccess(ITestResult result) {
        updateTestStatus(result, "1");
    }

    @Override
    public void onTestFailure(ITestResult result) {
        updateTestStatus(result, "0");
    }

    private void updateTestStatus(ITestResult result, String success) {
        Object testInstance = result.getInstance();
        if (testInstance instanceof TestingBotBaseTest) {
            AndroidDriver driver = ((TestingBotBaseTest) testInstance).driver;
            if (driver != null) {
                api.updateTest(
                    driver.getSessionId().toString(),
                    Map.of(
                        "success", success,
                        "name", result.getMethod().getMethodName()
                    )
                );
            }
        }
    }
}

Register the listener in your testng.xml: 

<suite name="Mobile Tests">
    <listeners>
        <listener class-name="TestingBotListener"/>
    </listeners>
    <!-- tests... -->
</suite>

Preparing your App

You do not need to install any code or plugin to run a test.
Below are some things that are necessary to successfully run a mobile test:

For Android:
  • Please supply the URL to your .apk or .aab file.
    Important: the .apk file needs to be a x86 build (x86 ABI) for Android emulators.
For iOS Real Device:
  • Please supply the URL to an .ipa file.
For iOS Simulator:
  • Please supply the URL to a .zip file that contains your .app
  • The .app needs to be an iOS Simulator build:
    • Create a Simulator build:
      xcodebuild -sdk iphonesimulator -configuration Debug
    • Zip the .app bundle:
      zip -r MyApp.zip MyApp.app

Additional Options

Appium provides a lot of options to configure your test.

Some important options that might help:

For Android:
  • appActivity and appPackage: by default, Appium will try to extract the main Activity from your apk. If this fails, you can supply your own with these options.
  • chromeOptions: additional chromedriver options you can supply.
  • otherApps: a JSON array of other apps that need to be installed, in URL format.
For Android & iOS:
  • locale: the language of the simulator/device (ex: fr_CA)

    This sets the locale on the entire device/simulator, except on physical iOS devices. For real iOS devices, we will pass -AppleLocale as argument to the app.

  • newCommandTimeout: How long (in seconds) Appium will wait for a new command from the client before assuming the client quit and ending the session
Was this page helpful?

Looking for More Help?

Have questions or need more information?
You can reach us via the following channels:

logo TestingBot