@phdthesis{14050,
  author = {Shuai Wang},
  title = {Systematic Product Line Testing: Methodologies, Automation, and Industrial Application},
  abstract = {Testing products in a cost-efficient way remains an attractive topic for product line testing in both academia and industry, which can be addressed by employing systematic and automated approaches utilizing models such as feature models. Driven by the needs of our industrial problem for testing Video Conferencing Systems (VCSs) product line developed by Cisco Systems, Norway, cost-effective testing can be further formulated into four main problems, including: 1) Test Selection: Automatically and systemically select a set of relevant test cases for a product from the entire suite available for a product line; 2) Test Minimization: Minimizing the test suite obtained by the selection to eliminate redundant test cases for reducing the cost of testing (e.g., execution time) while preserving high effectiveness (e.g., fault detection capability); 3) Test Generation: Automatically and systemically generate test cases when new functionalities are introduced to the product line by the product; and 4) Test Prioritization: Prioritizing the minimized test suite with the aim at balancing the trade-off between cost and effectiveness. To tackle the above-mentioned four challenges, this thesis presents a set of methodologies on industrial systems for cost- effective testing of a product in a product line, namely Product Line Model-based Testing Methodologies (PL-MTM), which includes: 1) a systematic and automated test selection methodology using feature model; 2) an automated search-based test minimization approach; 3) an automated and systematic test generation methodology using feature model and 4) an automated search-based test prioritization approach. For test selection, the contribution of this thesis is a systematic and automated test selection methodology using: 1) Feature Model for Testing (FM_T) to capture commonalities and variabilities of a product line; 2) Component Family Model for Testing (CFM_T) to model the structure of test case repository; 3) A tool to automatically build restrictions from CFM_T to FM_T and traces from CFM_T to the actual test cases. Using our methodology, a test engineer is only required to select relevant features through FM_T at a higher level of abstraction for a product and the corresponding test cases will be obtained automatically. We evaluate our methodology by applying it to a VCS product line called Saturn with seven commercial products and the results showed that our methodology can significantly reduce cost measured as test selection time and at the same time achieves higher effectiveness (feature coverage, feature pairwise coverage and fault detection) as compared with the current manual process. Moreover, we conducted a questionnaire-based study to solicit the views of test engineers who are involved in developing FM_T and CFM_T. The results showed that test engineers are positive about adopting our methodology in their current practice. Furthermore, we conducted a thorough controlled experiment to further collect evidence about the benefits of our proposed automated methodology. The results showed that our methodology is cost-effective as compared with the manual approach.For test minimization, the contribution of this thesis is a search-based test minimization technique. More specifically, the minimization problem is as a search problem and we defined a fitness function considering various optimization objectives based on the required cost-effective objectives (e.g., feature pairwise coverage and fault detection capability). To assess the performance of our fitness function, we conducted an extensive empirical evaluation by investigating the fitness function with three weight-based Genetic Algorithms (GAs) and seven multi-objective search algorithms using an industrial case study and 500 artificial problems inspired from the industrial case study. The results showed that Random-Weighted Genetic Algorithm (RWGA) significantly outperforms the other algorithms since RWGA can balance all the objectives together by dynamically updating weights at each generation. Based on the results of our empirical evaluation, we also implemented a tool called TEst Minimization using Search Algorithms (TEMSA) to support test minimization using various search algorithms in the context of product lines.For test generation, the contribution of this thesis is an extended systematic and automated test generation methodology using FM_T and Component Family Model for Behaviors (CFM_B). FM_T captures testing functionalities of a product line, whereas CFM_B provides an abstraction layer on top of the configurable state machines. With our current methodology, a test engineer doesn\’t need to acquire expertise on behavioral modeling and can simply configure models for a product by selecting features in FM_T and configuring provided attributes in CFM_B. The configured models are then given input to our model-based testing tool, TRansformation-based tool for Uml-baSed Testing (TRUST) for executable test case generation. We applied our extended methodology to a product line of video conferencing system developed by Cisco Systems, Norway. Results showed that the methodology significantly reduces the complexity of configuration; thereby significantly reducing required effort and cost (e.g., in terms of training). In addition, it does not require test engineers to have expertise in UML modelling, aspect- oriented modelling, and OCL specification and therefore eases the adoption in industry.For test prioritization, the contribution of this thesis is a search-based multi-objective test prioritization technique, considering multiple cost and effectiveness measures. In particular, our multi-objective optimization setup includes the minimization of execution cost (e.g., time), and the maximizing the number of prioritized test cases, feature pairwise coverage and fault detection capability. Based on cost-effectiveness measures, a novel fitness function is defined for such test prioritization problem. The fitness function is empirically evaluated together with three commonly used search algorithms (e.g., (1+1) Evolutionary algorithm (EA)) and Random Search as a comparison baseline based on the Cisco\’s industrial case study and 500 artificial designed problems. The results showed that (1+1) EA achieves the best performance for solving the test prioritization problem and it scales up to solve the problems of varying complexity.},
  year = {2015},
  publisher = {University of Oslo (UiO)},
}