Features
Start a free trial

Smart TV Testing

Run tests on physical Smart TV devices with TestingBot, using the open-source test framework Appium.

Currently TestingBot supports running automated tests on physical AppleTV 4k devices.

To get started with testing your Smart TV app in the cloud, you will need to follow these steps:

  • Upload your native Smart TV app, either to a public URL or using TestingBot Storage.
  • Create an Appium test script that will connect to TestingBot.
  • The results will be available in the TestingBot dashboard, together with a video of the test and logs.

Upload Smart TV App

To run Smart TV tests on TestingBot, you will first have to upload your .ipa file to TestingBot Storage.

Copy code
$ curl -X POST "https://api.testingbot.com/v1/storage" \
-u key:secret -F "file=@/path/to/app/file/tvos.ipa"

This call will return a unique identifier for your SmartTV app ({"app_url": "tb://..."}), which you can use in the capabilities in your test (see example below).

More information regarding this API call and other similar calls (update uploaded file, delete uploaded file, list uploaded files) are available in the TestingBot API documentation.

Note: uploads to TestingBot Storage are automatically deleted after 62 days.

Run your first SmartTV test

Once you've uploaded your Smart TV app, you can start with running your first test on the TestingBot SmartTV cloud.

TestingBot currently only supports running tests on physical AppleTV 4k devices. Please see the example tests below, where we use the TestingBot tvOS example app to input two numbers in the calculator and verify its sum.

Copy code
require 'rubygems'
require 'appium_lib'

caps = {}
caps['name'] = 'Ruby tvOS Example'
caps['deviceName'] = 'Apple TV 4K'
caps['platformName'] = 'tvOS'
caps['version'] = '17.4'
caps['app'] = 'https://github.com/testingbot/tvos-example/releases/download/1.0.0/tvos-example.ipa'
caps['realDevice'] = true

appium_driver = Appium::Driver.new({
    caps: caps,
    appium_lib: {
        server_url: "https://key:secret@hub.testingbot.com/wd/hub"
    }}, true)
driver = appium_driver.start_driver

wait = Selenium::WebDriver::Wait.new(:timeout => 30)
wait.until { driver.find_element(:accessibility_id, "inputField1").displayed? }

inputField1 = driver.find_element(:accessibility_id, "inputField1")
inputField1.click
inputField1.send_keys 5
driver.execute_script('mobile: pressButton', { name: 'down' })
driver.execute_script('mobile: pressButton', { name: 'select' })

driver.switch_to.active_element.send_keys 10

driver.execute_script('mobile: pressButton', { name: 'down' })
driver.execute_script('mobile: pressButton', { name: 'select' })
result = driver.find_element(:accessibility_id, "resultField")

if (!result.nil?) && (result.text.to_i == 15)
  puts "Test Passed"
else
  puts "Test Failed"
end
driver.quit
Copy code
const wdio = require('webdriverio');

const caps = {
    name: 'Node.js tvOS Example',
    deviceName: 'Apple TV 4K',
    platformName: 'tvOS',
    platformVersion: '17.4',
    app: 'https://github.com/testingbot/tvos-example/releases/download/1.0.0/tvos-example.ipa',
    realDevice: true
};

const options = {
    protocol: 'https',
    hostname: 'hub.testingbot.com',
    port: 443,
    path: '/wd/hub',
    user: 'key', // Replace 'key' with your TestingBot key
    key: 'secret', // Replace 'secret' with your TestingBot secret
    capabilities: caps
};

(async () => {
    // Start the Appium driver
    const driver = await wdio.remote(options);

    try {
        // Wait for the first input field to be displayed
        const inputField1 = await driver.$('~inputField1');
        await inputField1.waitForDisplayed({ timeout: 30000 });

        // Interact with the first input field
        await inputField1.click();
        await inputField1.setValue(5);

        // Navigate down and select
        await driver.execute('mobile: pressButton', { name: 'down' });
        await driver.execute('mobile: pressButton', { name: 'select' });

        // Send keys to the active element
        await driver.switchToParentFrame(); // Ensure focus is on the active frame
        await driver.activeElement().setValue(10);

        // Navigate down and select the result
        await driver.execute('mobile: pressButton', { name: 'down' });
        await driver.execute('mobile: pressButton', { name: 'select' });

        // Verify the result
        const result = await driver.$('~resultField');
        const resultText = await result.getText();

        if (resultText && parseInt(resultText, 10) === 15) {
            console.log('Test Passed');
        } else {
            console.log('Test Failed');
        }
    } catch (err) {
        console.error('Error occurred:', err);
    } finally {
        // Quit the driver
        await driver.deleteSession();
    }
})();
Copy code
from appium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Define capabilities
caps = {
    "name": "Python tvOS Example",
    "deviceName": "Apple TV 4K",
    "platformName": "tvOS",
    "platformVersion": "17.4",
    "app": "https://github.com/testingbot/tvos-example/releases/download/1.0.0/tvos-example.ipa",
    "realDevice": True
}

# Appium server URL
server_url = "https://key:secret@hub.testingbot.com/wd/hub"

# Initialize the driver
driver = webdriver.Remote(server_url, caps)

try:
    # Wait for the first input field to be displayed
    wait = WebDriverWait(driver, 30)
    input_field1 = wait.until(EC.presence_of_element_located((By.ACCESSIBILITY_ID, "inputField1")))

    # Interact with the first input field
    input_field1.click()
    input_field1.send_keys("5")

    # Navigate down and select the second input field
    driver.execute_script('mobile: pressButton', {'name': 'down'})
    driver.execute_script('mobile: pressButton', {'name': 'select'})

    # Send keys to the active element (second input field)
    driver.switch_to.active_element.send_keys("10")

    # Navigate down and select the result field
    driver.execute_script('mobile: pressButton', {'name': 'down'})
    driver.execute_script('mobile: pressButton', {'name': 'select'})

    # Verify the result
    result_field = driver.find_element(By.ACCESSIBILITY_ID, "resultField")
    result_text = result_field.text

    if result_text and int(result_text) == 15:
        print("Test Passed")
    else:
        print("Test Failed")

except Exception as e:
    print(f"An error occurred: {e}")

finally:
    # Quit the driver
    driver.quit()
Copy code
import io.appium.java_client.AppiumDriver;
import io.appium.java_client.MobileBy;
import io.appium.java_client.remote.MobileCapabilityType;
import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.remote.DesiredCapabilities;

import java.net.URL;
import java.time.Duration;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

public class TvOSTest {
    public static void main(String[] args) {
        AppiumDriver<WebElement> driver = null;

        try {
            // Set capabilities
            DesiredCapabilities caps = new DesiredCapabilities();
            caps.setCapability("name", "Java tvOS Example");
            caps.setCapability("deviceName", "Apple TV 4K");
            caps.setCapability("platformName", "tvOS");
            caps.setCapability("platformVersion", "17.4");
            caps.setCapability("app", "https://github.com/testingbot/tvos-example/releases/download/1.0.0/tvos-example.ipa");
            caps.setCapability("realDevice", true);

            URL serverUrl = new URL("https://key:secret@hub.testingbot.com/wd/hub");

            driver = new AppiumDriver<>(serverUrl, caps);

            // Wait for the first input field to be displayed
            WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));
            WebElement inputField1 = wait.until(ExpectedConditions.presenceOfElementLocated(MobileBy.AccessibilityId("inputField1")));

            inputField1.click();
            inputField1.sendKeys("5");

            // Navigate down and select the second input field
            driver.executeScript("mobile: pressButton", Map.of("name", "down"));
            driver.executeScript("mobile: pressButton", Map.of("name", "select"));

            // Send keys to the active element (second input field)
            WebElement activeElement = driver.switchTo().activeElement();
            activeElement.sendKeys("10");

            // Navigate down and select the result field
            driver.executeScript("mobile: pressButton", Map.of("name", "down"));
            driver.executeScript("mobile: pressButton", Map.of("name", "select"));

            WebElement resultField = driver.findElement(MobileBy.AccessibilityId("resultField"));
            String resultText = resultField.getText();

            if (resultText != null && Integer.parseInt(resultText) == 15) {
                System.out.println("Test Passed");
            } else {
                System.out.println("Test Failed");
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // Quit the driver
            if (driver != null) {
                driver.quit();
            }
        }
    }
}
Copy code
using OpenQA.Selenium;
using OpenQA.Selenium.Appium;
using OpenQA.Selenium.Appium.Enums;
using OpenQA.Selenium.Appium.iOS;
using OpenQA.Selenium.Support.UI;
using System;
using System.Collections.Generic;

class TvOSTest
{
    static void Main(string[] args)
    {
        AppiumDriver<IWebElement> driver = null;

        try
        {
            // Set capabilities
            var caps = new AppiumOptions();
            caps.AddAdditionalCapability(MobileCapabilityType.PlatformName, "tvOS");
            caps.AddAdditionalCapability(MobileCapabilityType.DeviceName, "Apple TV 4K");
            caps.AddAdditionalCapability(MobileCapabilityType.PlatformVersion, "17.4");
            caps.AddAdditionalCapability(MobileCapabilityType.App, "https://github.com/testingbot/tvos-example/releases/download/1.0.0/tvos-example.ipa");
            caps.AddAdditionalCapability("realDevice", true);
            caps.AddAdditionalCapability("name", "C# tvOS Example");

            // Initialize driver
            var serverUri = new Uri("https://key:secret@hub.testingbot.com/wd/hub");
            driver = new IOSDriver<IWebElement>(serverUri, caps);

            // Wait for the first input field to be displayed
            var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
            var inputField1 = wait.Until(d => d.FindElement(MobileBy.AccessibilityId("inputField1")));

            // Interact with the first input field
            inputField1.Click();
            inputField1.SendKeys("5");

            // Navigate down and select the second input field
            driver.ExecuteScript("mobile: pressButton", new Dictionary<string, string> { { "name", "down" } });
            driver.ExecuteScript("mobile: pressButton", new Dictionary<string, string> { { "name", "select" } });

            // Send keys to the active element (second input field)
            var activeElement = driver.SwitchTo().ActiveElement();
            activeElement.SendKeys("10");

            // Navigate down and select the result field
            driver.ExecuteScript("mobile: pressButton", new Dictionary<string, string> { { "name", "down" } });
            driver.ExecuteScript("mobile: pressButton", new Dictionary<string, string> { { "name", "select" } });

            // Verify the result
            var resultField = driver.FindElement(MobileBy.AccessibilityId("resultField"));
            var resultText = resultField.Text;

            if (!string.IsNullOrEmpty(resultText) && int.Parse(resultText) == 15)
            {
                Console.WriteLine("Test Passed");
            }
            else
            {
                Console.WriteLine("Test Failed");
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine($"An error occurred: {ex.Message}");
        }
        finally
        {
            // Quit the driver
            driver?.Quit();
        }
    }
}

View SmartTV Test Results

Once your test has finished running, you will see a new test entry in the TestingBot dashboard.. If you click the test, you will see detailed information about the test, including a video recording of the test, logs and screenshots.

You can also use the TestingBot API to programmatically fetch the test results.