@misc{14325,
  keywords = {Uncertainty-wise Testing; Test Case Prioritization; Multi-objective Search},
  author = {Man Zhang and Yan Li and Shaukat Ali and Tao Yue},
  title = {Uncertainty-Wise and Time-Aware Test Case Prioritization for Cyber-Physical Systems},
  abstract = {Context: Complex systems (e.g., Cyber-Physical Systems) that interact with the real world, behave in an unstipulated manner while operating in uncertain environments. Testing such systems in uncertainty is a big challenge. Devising uncertainty-wise testing solutions can be considered as a mandate for dealing with this challenge. Though uncertainty-wise testing is gaining attention in the last few years, industry-strengthening solutions are still missing.Objective: Our objective is to propose uncertainty-wise test case prioritization approaches that can significantly improve the cost-effectiveness of test case execution to maximize the occurrence of uncertainty.Method: In this paper, we identified and defined twenty uncertainty-wise, search-based, multi-objective test case prioritization problems. To solve these problems with multi-objective search algorithms, we defined twenty corresponding fitness functions based on various cost, effectiveness, and uncertainty measures.Results: We performed a large-scale empirical evaluation to evaluate the well-known multi-objective search algorithms, i.e., NSGA-II, MOCell, SPEA2 and CellDE, and Random Search (RS), with five different use cases from two industrial CPS case studies to solve the twenty prioritization problems. Results showed that all the selected search algorithms significantly outperformed RS signifying that our identified test prioritization problems are complex. When comparing search algorithms, we observed that different search algorithms performed best regarding providing the best solutions for different uncertainty-wise prioritization problems. Based on the results, we provide recommendations to select search algorithms for different prioritization problems under different time budgets. Conclusion: This paper presented uncertainty-wise and time-aware test case prioritization approaches, which were specially developed to improve the cost and effectiveness of test case execution and at the same time maximizing the occurrence of uncertainties during CPS testing},
  year = {2017},
  publisher = {Simula Research Laboratory},
}