TestNG Automated App Testing
See our TestNG example repository for a simple example on how to run TestNG mobile app tests.
Installation
First, make sure you install the necessary dependencies to run a test. For this example, we'll be using TestNG and the Appium Java Client.
Add the following dependencies to your pom.xml:
<dependencies>
<!-- TestNG -->
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>7.12.0</version>
<scope>test</scope>
</dependency>
<!-- Appium Java Client -->
<dependency>
<groupId>io.appium</groupId>
<artifactId>java-client</artifactId>
<version>10.1.1</version>
<scope>test</scope>
</dependency>
</dependencies>
Note: Appium Java Client 10.x requires Java 11 or higher.
TestNG Example
The example below shows a complete TestNG test that connects to the TestingBot hub and runs a test on a real device. In this test, we'll test a sample calculator app by entering values into two input fields and verifying the sum.
import io.appium.java_client.AppiumBy;
import io.appium.java_client.android.AndroidDriver;
import io.appium.java_client.android.options.UiAutomator2Options;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.Duration;
import java.util.Map;
public class AndroidAppTest {
private AndroidDriver driver;
@BeforeMethod
public void setUp() throws MalformedURLException {
// Configure capabilities using UiAutomator2Options
UiAutomator2Options options = new UiAutomator2Options()
.setPlatformName("Android")
.setPlatformVersion("16.0")
.setDeviceName("Pixel 9")
.setApp("tb://<your-app-id>")
.setAutomationName("UiAutomator2");
// Add TestingBot-specific options
options.setCapability("tb:options", Map.of(
"key", "YOUR_TB_KEY",
"secret", "YOUR_TB_SECRET",
"name", "Android Calculator Test",
"build", "TestNG Build 1"
));
// Connect to TestingBot hub
driver = new AndroidDriver(
new URL("https://hub.testingbot.com/wd/hub"),
options
);
}
@Test
public void testCalculatorSum() {
WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));
// Enter first number
WebElement inputA = wait.until(
ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
);
inputA.sendKeys("10");
// Enter second number
WebElement inputB = wait.until(
ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
);
inputB.sendKeys("5");
// Verify the sum
WebElement sum = wait.until(
ExpectedConditions.visibilityOfElementLocated(AppiumBy.accessibilityId("sum"))
);
Assert.assertEquals(sum.getText(), "15");
}
@AfterMethod
public void tearDown() {
if (driver != null) {
driver.quit();
}
}
}
import io.appium.java_client.AppiumBy;
import io.appium.java_client.ios.IOSDriver;
import io.appium.java_client.ios.options.XCUITestOptions;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.Duration;
import java.util.Map;
public class IOSAppTest {
private IOSDriver driver;
@BeforeMethod
public void setUp() throws MalformedURLException {
// Configure capabilities using XCUITestOptions
XCUITestOptions options = new XCUITestOptions()
.setPlatformName("iOS")
.setPlatformVersion("26.0")
.setDeviceName("iPhone 17")
.setApp("tb://<your-app-id>")
.setAutomationName("XCUITest");
// Add TestingBot-specific options
options.setCapability("tb:options", Map.of(
"key", "YOUR_TB_KEY",
"secret", "YOUR_TB_SECRET",
"name", "iOS Calculator Test",
"build", "TestNG Build 1"
));
// Connect to TestingBot hub
driver = new IOSDriver(
new URL("https://hub.testingbot.com/wd/hub"),
options
);
}
@Test
public void testCalculatorSum() {
WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));
// Enter first number
WebElement inputA = wait.until(
ExpectedConditions.elementToBeClickable(AppiumBy.iOSNsPredicateString("name == 'inputA' AND type == 'XCUIElementTypeTextField'"))
);
inputA.clear(); // the field starts with "0"
inputA.sendKeys("10");
// Enter second number
WebElement inputB = wait.until(
ExpectedConditions.elementToBeClickable(AppiumBy.iOSNsPredicateString("name == 'inputB' AND type == 'XCUIElementTypeTextField'"))
);
inputB.clear();
inputB.sendKeys("5");
// Verify the sum
WebElement sum = wait.until(
ExpectedConditions.visibilityOfElementLocated(AppiumBy.iOSNsPredicateString("name == 'sum' AND type == 'XCUIElementTypeTextField'"))
);
Assert.assertEquals(sum.getText(), "15");
}
@AfterMethod
public void tearDown() {
if (driver != null) {
driver.quit();
}
}
}
Always call driver.quit() in your @AfterMethod to properly end the session. Otherwise, the test will continue running until it times out.
Reusable Base Test Class
For cleaner test organization, you can create a reusable base class that handles driver setup and teardown. Tests can then extend this base class and focus only on test logic.
Base Class
import io.appium.java_client.android.AndroidDriver;
import io.appium.java_client.android.options.UiAutomator2Options;
import org.testng.annotations.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
public class TestingBotBaseTest {
protected AndroidDriver driver;
@BeforeMethod(alwaysRun = true)
@Parameters({"platformVersion", "deviceName", "app"})
public void setUp(
@Optional("16.0") String platformVersion,
@Optional("Pixel 9") String deviceName,
@Optional("tb://<your-app-id>") String app
) throws MalformedURLException {
// Get credentials from environment or use defaults
String key = System.getenv("TB_KEY");
String secret = System.getenv("TB_SECRET");
if (key == null || secret == null) {
throw new RuntimeException("TB_KEY and TB_SECRET environment variables must be set");
}
// Configure capabilities
UiAutomator2Options options = new UiAutomator2Options()
.setPlatformName("Android")
.setPlatformVersion(platformVersion)
.setDeviceName(deviceName)
.setApp(app)
.setAutomationName("UiAutomator2");
options.setCapability("tb:options", Map.of(
"key", key,
"secret", secret,
"name", this.getClass().getSimpleName()
));
driver = new AndroidDriver(
new URL("https://hub.testingbot.com/wd/hub"),
options
);
}
@AfterMethod(alwaysRun = true)
public void tearDown() {
if (driver != null) {
driver.quit();
}
}
}
Test Class Using the Base
import io.appium.java_client.AppiumBy;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.time.Duration;
public class CalculatorTest extends TestingBotBaseTest {
@Test
public void testAddition() {
WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));
WebElement inputA = wait.until(
ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputA"))
);
inputA.sendKeys("10");
WebElement inputB = wait.until(
ExpectedConditions.elementToBeClickable(AppiumBy.accessibilityId("inputB"))
);
inputB.sendKeys("5");
WebElement sum = wait.until(
ExpectedConditions.visibilityOfElementLocated(AppiumBy.accessibilityId("sum"))
);
Assert.assertEquals(sum.getText(), "15");
}
}
TestNG XML Configuration
Use a testng.xml file to pass parameters and run tests on multiple devices:
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
<suite name="Mobile App Tests" parallel="tests" thread-count="2">
<test name="Android Pixel 9">
<parameter name="platformVersion" value="16.0"/>
<parameter name="deviceName" value="Pixel 9"/>
<parameter name="app" value="tb://your-app-id"/>
<classes>
<class name="CalculatorTest"/>
</classes>
</test>
<test name="Android Pixel 8">
<parameter name="platformVersion" value="14.0"/>
<parameter name="deviceName" value="Pixel 8"/>
<parameter name="app" value="tb://your-app-id"/>
<classes>
<class name="CalculatorTest"/>
</classes>
</test>
</suite>
Run the suite with Maven
To make mvn test run this testng.xml, point the Maven Surefire plugin at it, and pin Surefire to 3.5.x:
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.5.6</version>
<configuration>
<suiteXmlFiles>
<suiteXmlFile>testng.xml</suiteXmlFile>
</suiteXmlFiles>
</configuration>
</plugin>
</plugins>
</build>
Surefire 3.6.0 and later run TestNG through the JUnit Platform and reject suiteXmlFiles with
Parameter "suiteXmlFiles" is not supported since 3.6.0, so mvn test fails before any test starts.
Either stay on 3.5.x as above, or drop testng.xml from the Maven build and select tests with Surefire's
groups, includes or excludes instead.
Uploading Your App
Please see our Upload Mobile App documentation to find out how to upload your app to TestingBot for testing.
Specify Browsers & Devices
To run your tests on TestingBot, you need to configure the device and platform you want to test on. The Appium Java client provides type-safe Options classes for this purpose.
Using Options Classes (Recommended)
For Android testing, use UiAutomator2Options. For iOS testing, use XCUITestOptions:
UiAutomator2Options options = new UiAutomator2Options()
.setPlatformName("Android")
.setPlatformVersion("16.0")
.setDeviceName("Pixel 9")
.setApp("tb://<your-app-id>")
.setAutomationName("UiAutomator2");
// Add TestingBot credentials
options.setCapability("tb:options", Map.of(
"key", "YOUR_TB_KEY",
"secret", "YOUR_TB_SECRET",
"name", "My Test",
"build", "Build 1"
));
AndroidDriver driver = new AndroidDriver(
new URL("https://hub.testingbot.com/wd/hub"),
options
);
XCUITestOptions options = new XCUITestOptions()
.setPlatformName("iOS")
.setPlatformVersion("26.0")
.setDeviceName("iPhone 17")
.setApp("tb://<your-app-id>")
.setAutomationName("XCUITest");
// Add TestingBot credentials
options.setCapability("tb:options", Map.of(
"key", "YOUR_TB_KEY",
"secret", "YOUR_TB_SECRET",
"name", "My Test",
"build", "Build 1"
));
IOSDriver driver = new IOSDriver(
new URL("https://hub.testingbot.com/wd/hub"),
options
);
Using BaseOptions (Generic)
For more flexibility or when working with multiple platforms, you can use BaseOptions:
import io.appium.java_client.remote.options.BaseOptions;
BaseOptions options = new BaseOptions()
.setPlatformName("Android")
.setAutomationName("UiAutomator2")
.amend("appium:deviceName", "Pixel 9")
.amend("appium:platformVersion", "16.0")
.amend("appium:app", "tb://<your-app-id>");
options.setCapability("tb:options", Map.of(
"key", "YOUR_TB_KEY",
"secret", "YOUR_TB_SECRET"
));
AppiumDriver driver = new AppiumDriver(
new URL("https://hub.testingbot.com/wd/hub"),
options
);
Required Capabilities
To specify which device you want to test on, you need to provide:
-
platformName- The mobile OS (Android or iOS) -
appium:platformVersion- The OS version (e.g., "16.0", "26.0") -
appium:deviceName- The device name (e.g., "Pixel 9", "iPhone 17") -
appium:app- The app to test (URL or tb:// reference) -
appium:automationName- The automation driver ("UiAutomator2" for Android, "XCUITest" for iOS)
Use the device picker below to generate the configuration for your target device:
Testing Internal Websites
We've built TestingBot Tunnel, to provide you with a secure way to run tests against your staged/internal webapps.
Please see our TestingBot Tunnel documentation for more information about this easy to use tunneling solution.
The example below shows how to easily run a TestNG test with our Tunnel:
1. Download our tunnel and start the tunnel:
java -jar testingbot-tunnel.jar key secret
2. Update your test to connect through the tunnel instead of directly to the TestingBot hub.
Change the URL from hub.testingbot.com to localhost:4445:
// Connect through the tunnel instead of directly to TestingBot
AndroidDriver driver = new AndroidDriver(
new URL("http://localhost:4445/wd/hub"),
options
);
Your test traffic will now go securely through the tunnel, allowing you to test apps that access internal resources.
Mark tests as passed/failed
TestingBot cannot automatically determine if your test passed or failed - that depends on your test assertions. To report test results to TestingBot, use our REST API.
First, add the TestingBot Java client to your dependencies:
<dependency>
<groupId>com.testingbot</groupId>
<artifactId>testingbotrest</artifactId>
<version>1.1.0</version>
</dependency>
Then use it in your test to report the result:
import com.testingbot.testingbotrest.TestingbotREST;
import java.util.Map;
@AfterMethod
public void tearDown() {
if (driver != null) {
// Report test result to TestingBot
TestingbotREST api = new TestingbotREST(
"YOUR_TB_KEY",
"YOUR_TB_SECRET"
);
api.updateTest(
driver.getSessionId().toString(),
Map.of(
"success", "1", // "1" for passed, "0" for failed
"name", "Calculator Test"
)
);
driver.quit();
}
}
For automatic result reporting, you can use a TestNG listener:
import io.appium.java_client.android.AndroidDriver;
import org.testng.ITestListener;
import org.testng.ITestResult;
import com.testingbot.testingbotrest.TestingbotREST;
import java.util.Map;
public class TestingBotListener implements ITestListener {
private TestingbotREST api;
public TestingBotListener() {
this.api = new TestingbotREST(
System.getenv("TB_KEY"),
System.getenv("TB_SECRET")
);
}
@Override
public void onTestSuccess(ITestResult result) {
updateTestStatus(result, "1");
}
@Override
public void onTestFailure(ITestResult result) {
updateTestStatus(result, "0");
}
private void updateTestStatus(ITestResult result, String success) {
Object testInstance = result.getInstance();
if (testInstance instanceof TestingBotBaseTest) {
AndroidDriver driver = ((TestingBotBaseTest) testInstance).driver;
if (driver != null) {
api.updateTest(
driver.getSessionId().toString(),
Map.of(
"success", success,
"name", result.getMethod().getMethodName()
)
);
}
}
}
}
Register the listener in your testng.xml:
<suite name="Mobile Tests">
<listeners>
<listener class-name="TestingBotListener"/>
</listeners>
<!-- tests... -->
</suite>
Preparing your App
You do not need to install any code or plugin to run a test.
Below are some things that are necessary to successfully run a mobile test:
- Please supply the URL to your
.apkor.aabfile.
Important: the.apkfile needs to be a x86 build (x86 ABI) for Android emulators.
- Please supply the URL to an
.ipafile.
- Please supply the URL to a
.zipfile that contains your.app -
The
.appneeds to be an iOS Simulator build:
- Create a Simulator build:
xcodebuild -sdk iphonesimulator -configuration Debug - Zip the
.appbundle:zip -r MyApp.zip MyApp.app
- Create a Simulator build:
Additional Options
Appium provides a lot of options to configure your test.
Some important options that might help:
For Android:- appActivity and appPackage: by default, Appium will try to extract the main Activity from your apk. If this fails, you can supply your own with these options.
- chromeOptions: additional chromedriver options you can supply.
- otherApps: a JSON array of other apps that need to be installed, in URL format.
-
locale: the language of the simulator/device (ex:
fr_CA)This sets the locale on the entire device/simulator, except on physical iOS devices. For real iOS devices, we will pass
-AppleLocaleas argument to the app. - newCommandTimeout: How long (in seconds) Appium will wait for a new command from the client before assuming the client quit and ending the session