Get log entries for individual test case

Class library usage, coding and language questions.
dj91
Posts: 4
Joined: Mon Apr 24, 2017 10:33 am

Get log entries for individual test case

Post by dj91 » Tue Apr 25, 2017 8:44 am

Hello,

I am trying to connect Ranorex with TestRail. So far I can only report whether the test case execution was successful or not.

What I really want is something like this:
1.jpg
1.jpg (232.32 KiB) Viewed 1013 times
Well, this doesn't work yet because I want all the log entries and data specific to this case.

What I have so far is this:

- A "TestRailReporter", which implements IReportLogger (see below)
-- The Start() method clears a list of messages, so that for every test case I should only have the log messages reported during the execution of the test case.
-- The End() method then takes the list of messages, builds a comment with Markdown and then sends the information to TestRail.
- Attached the logger in Program.cs:

Code: Select all

Report.AttachLogger(new TestRailReporter());
- Two code modules which are called in the setup and teardown sections respectively: PrepareForReporting, which calls Report.Start() and ReportToTestManagement, which calls Report.End()

I followed this (outdated) blog post: First google result for "Customizing Ranorex Reports" (sorry, not allowed to post links on here).

I have set up three identical test cases with a code module that always fails. This is my result:

On the first test case I get all the information logged before the test case even started - and also a lot of information about the second test case.

The second test case has already lost a lot of information to the first case and also contains information about the last case.

The last test case only gets the information "Failed" and all the info happening AFTER the test run.

The funny thing is:
2.jpg
2.jpg (133.07 KiB) Viewed 1013 times
The stuff in the yellow box should be called BEFORE the red box, as it is called in the teardown section of the test case.

BUT, if I remove my code modules which trigger Report.Start() and Report.End() from the Setup/Teardown sections and add them right into the test body, everything works fine. I assume Ranorex doesn't have test case information available during setup?

Anyways, removing them from the setup/teardown sections is NOT a solution as I also want to log what happens during setup/teardown.

We are currently evaluating test automation tools and so far I liked Ranorex the most as it seems to be the only application which works very well with our UI elements (for example, other tools recognize a ribbon toolbar, but not the buttons in it. With Ranorex I can actually check if a button in the ribbon is enabled.), but the integration with our test management system can be the showstopper for our decision for or against a test automation tool.

This is the TestRailReporter:

Code: Select all

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Text;

using Gurock.TestRail;
using Newtonsoft.Json.Linq;
using Ranorex;
using Ranorex.Core;
using Ranorex.Core.Reporting;
using Ranorex.Core.Testing;

namespace MyTestSuite
{
	/// <summary>
	/// Description of TestRailReporter.
	/// </summary>
	public class TestRailReporter : IReportLogger
	{
		private System.DateTime start;
		private APIClient client;
		private ITestSuite testSuite;
		private ITestContainerActivity testRun;
		private ITestContainer testCase;
		private int testRailTestRunId;
		private string testRailTestRunName;
		private string logLocation;
		private List<LogEntry> items;
		
		public TestRailReporter()
		{
			this.client = new APIClient("URL of our testrail instance");
        	this.client.User = "[email protected]";
        	this.client.Password = "MyApiKey";
        	
        	start = System.DateTime.Now;
			items = new List<LogEntry>();
		}
		
		public bool PreFilterMessages {
			get {
				return true;
			}
		}
		
		public void Start()
		{
			items.Clear();
			items.Add(new LogEntry() {
			          	Message = "Items cleared"
			          });
			
        	testSuite = TestSuite.Current;
        	testRun = TestReport.CurrentTestContainerActivity;
        	testRailTestRunId = testSuite.Parameters.ContainsKey("RunID") ? int.Parse(testSuite.Parameters["RunID"]) : 0;
        	
        	testCase = testRun.TestContainer;
        	
        	items.Add(new LogEntry() {
        	          	Message = "TS Name: "+testSuite.Name
        	          });
        	
        	items.Add(new LogEntry() {
        	          	Message = "TC Name: "+testCase.Name
        	          });
        	
        	
        	if(testSuite.Parameters.ContainsKey("RunID"))
        	{
        		testRailTestRunId = int.Parse(testSuite.Parameters["RunID"]);
        		
        		var testRunResponse = (JObject) client.SendGet("get_run/"+testSuite.Parameters["RunID"]);
        		
        		testRailTestRunName = testRunResponse.Property("name").Value.ToString();
        	}
        	//Report.Info("TEST 2");
        	
        	var testRunName = testRailTestRunName;
        	
        	foreach(var c in Path.GetInvalidFileNameChars())
    		{
    			testRunName = testRunName.Replace(c.ToString(), "_");
    		}
        	
        	logLocation = @"U:\60_TestAutomation\Logs\"+start.ToString("yyyy-MM-dd HH-mm-ss")+" "+testSuite.Name+" - "[email protected]"\"[email protected]"\";
        	
        	if(!Directory.Exists(logLocation)) {
        		Directory.CreateDirectory(logLocation);
        	}
        	
        	items.Add(new LogEntry() {
        	          	Message = "Log location: "+logLocation
        	          });
        	
        	//Report.Info(testRailTestRunName);
        	//Report.Info(logLocation);
        	//Report.Info("END: TestRailReporter.Start()");
		}
		
		public void End()
		{
			Report.Info("START: TestRailReporter.End()");
			Report.Info(testCase == null ? "testCase is null" : "testcase is not null: "+testCase.Name);
			
			if(!testCase.IsTestCase)
        	{
        		// We don't care about smart folders, only test cases
        		Report.Info("Smart folder results are not reported to TestRail");
        		return;
        	}
			
			if(!testCase.Parameters.ContainsKey("TestCase"))
        	{
        		// Only test cases with a TestRail Case ID are relevant
        		Report.Warn("Test case '"+testCase.Name+"' has no associated TestRail case ID.");
        		Report.Info("Add a parameter 'TestCase' with a comma-separated list of TestRail case IDs, e.g. 'C18, C19,C20'");
        		return;
        	}
			
			var testCaseIds = testCase.Parameters["TestCase"].Split(',').ToList().Select(i => i.Trim()).ToList();
        	
        	var result = 3; // this value is NOT allowed!
        	
        	if(testRun.Status == ActivityStatus.Success)
        	{
        		result = 1; // Success
        	}
        	else if(testRun.Status == ActivityStatus.Failed)
        	{
        		result = 5; // Failed
        	}
        	
        	var commentBuilder = new StringBuilder("This test has been executed by Ranorex\n\n")
        		.Append("||| :Field | :Value\n")
        		.Append("|| Run configuration:| " + testSuite.SelectedRunConfig.Name+"\n")
        		.Append("|| Test case: | "+testCase.Name+"\n")
        		.Append("|| Test case coverage: | "+String.Join(", ", testCaseIds)+"\n")
        		.Append("|| Parameters: | ");
        		        
        	var prefix = string.Empty;
	        foreach(var param in testCase.Parameters)
	        {
	        	commentBuilder.Append(prefix+param.Key+" = "+(param.Value == null ? "(null)" : param.Value)+"\n");
	        	
	        	prefix = "|| | ";
	        }
	        
	        commentBuilder.Append("\n\n\n");
	        
	        commentBuilder.Append("||| :Time stamp | :Report level | :Category | :Message | :Data\n");
	        
	        foreach(var i in items)
	        {
	        	commentBuilder.Append("|| "+i.Timestamp+" | "+i.ReportLevel.Name + " | " + i.Category + " | " + i.Message + " | " + i.DataPath + "\n");
	        }
	        
	        var comment = commentBuilder.ToString();
        	
        	foreach(var testCaseId in testCaseIds)
        	{
        		var elapsed = (testRun.ElapsedTime.Milliseconds/1000);
        		
        		if(elapsed < 1) {
        			elapsed = 1;
        		}
        		
        		var data = new Dictionary<string, string>();
	        	data.Add("status_id", result.ToString());
	        	data.Add("elapsed", elapsed+" s");
	        	data.Add("comment", comment);
	        	//data.Add("custom_automation_log_file", "idk");//PrepareReportingForTestRun.PathToLogFile);
	        	
	        	var targetNode = testRailTestRunId > 0 ? "add_result_for_case/"+testRailTestRunId : "add_result";
	        	
	        	var postResultResponse = client.SendPost(targetNode + "/"+testCaseIds[0].Substring(1), data);
	        	Report.Info("Test result reported to TestRail:\n\n"+postResultResponse.ToString());
        	}
        	
			testCase = null;
			
			Report.Info("END: TestRailReporter.End()");
		}
		
		public void LogText(Ranorex.ReportLevel level, string category, string message, bool escape, System.Collections.Generic.IDictionary<string, string> metaInfos)
		{
			var logEntry = new LogEntry {
				Timestamp = GetTimeStamp(),
				ReportLevel = level,
				Category = category,
				Message = message
			};
			
			items.Add(logEntry);
			          
		}
		
		public void LogData(Ranorex.ReportLevel level, string category, string message, object data, System.Collections.Generic.IDictionary<string, string> metaInfos)
		{
			string dataPath = null;
			
			if(data is Bitmap)
			{
				dataPath = logLocation+Guid.NewGuid()+".png";
				((Bitmap)data).Save(dataPath, ImageFormat.Png);
				Report.Info(dataPath);
			}
			
			var logEntry = new LogEntry {
				Timestamp = GetTimeStamp(),
				ReportLevel = level,
				Category = category,
				Message = message,
				DataPath = dataPath
			};
		}
		
		private string GetTimeStamp()
        {
			//return testRun.ElapsedTime.ToString("HH:mm:ss.fff", System.Globalization.CultureInfo.InvariantCulture);
            return System.DateTime.Now.ToString("HH:mm:ss.fff",
                                                System.Globalization.CultureInfo.InvariantCulture);
        }
	}
	
	class LogEntry
	{
		public String Timestamp {get;set;}
		public ReportLevel ReportLevel {get;set;}
		public string Category {get;set;}
		public string Message {get;set;}
		public string DataPath {get;set;}
	}
}

dj91
Posts: 4
Joined: Mon Apr 24, 2017 10:33 am

Re: Get log entries for individual test case

Post by dj91 » Tue Apr 25, 2017 2:28 pm

From my understanding, [SETUP] doesn't know about its test case, so I have shifted all my logic to the End() method of my IReportLogger. This got me much closer to what I want.

However, two problems remain:

- The report for my first test case also has all the report information of what happens in Program.cs and in the [SETUP] section of my test suite:
3.jpg
3.jpg (244.24 KiB) Viewed 1004 times
- The report for the next test case contains the final log information of the previous case:
4.jpg
4.jpg (205.01 KiB) Viewed 1004 times
- - This last information is then obviously missing in the final test case report

The new version of my TestRailReporter:
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Text;

using Gurock.TestRail;
using Newtonsoft.Json.Linq;
using Ranorex;
using Ranorex.Core;
using Ranorex.Core.Reporting;
using Ranorex.Core.Testing;

namespace Test
{
	/// <summary>
	/// Description of TestRailReporter.
	/// </summary>
	public class TestRailReporter : IReportLogger
	{
		public static System.DateTime StartTime {get;set;}
		public static APIClient Client {get;set;}
		
		private List<LogEntry> items;
		private int reportNumber;
		
		public TestRailReporter()
		{
			Client = new APIClient("URL to testrail instance");
        	Client.User = "[email protected]";
        	Client.Password = "my-api-key";
        	
        	StartTime = System.DateTime.Now;
			items = new List<LogEntry>();
			reportNumber = 0;
		}
		
		public bool PreFilterMessages {
			get {
				return true;
			}
		}
		
		public void Start()
		{
			items.Clear();
		}
		
		public void End()
		{
			var testSuite = TestSuite.Current;
        	var testRun = TestReport.CurrentTestContainerActivity;
        	
        	var testCase = testRun.TestContainer;
        	
        	var testRailTestRunName = string.Empty;
        	var testRailTestRunId = testSuite.Parameters.ContainsKey("RunID") ? int.Parse(testSuite.Parameters["RunID"]) : 0;

			if(!testCase.IsTestCase)
        	{
        		// We don't care about smart folders, only test cases
        		//Report.Info("Smart folder results are not reported to TestRail");
        		return;
        	}
			
			if(!testCase.Parameters.ContainsKey("TestCase"))
        	{
        		// Only test cases with a TestRail Case ID are relevant
        		//Report.Warn("Test case '"+testCase.Name+"' has no associated TestRail case ID.");
        		//Report.Info("Add a parameter 'TestCase' with a comma-separated list of TestRail case IDs, e.g. 'C18, C19,C20'");
        		return;
        	}
        	
			reportNumber++;
			
        	if(testRailTestRunId > 0)
        	{
        		var testRunResponse = (JObject) Client.SendGet("get_run/"+testSuite.Parameters["RunID"]);
        		
        		testRailTestRunName = testRunResponse.Property("name").Value.ToString();
        	}
        	
        	var testRunName = testRailTestRunName;
        	
        	foreach(var c in Path.GetInvalidFileNameChars())
    		{
    			testRunName = testRunName.Replace(c.ToString(), "_");
    		}
        	
        	var logLocation = @"U:\60_TestAutomation\Logs\"+StartTime.ToString("yyyy-MM-dd HH-mm-ss")+" "+testSuite.Name+" - "[email protected]"\"+reportNumber+" - "[email protected]"\";
        	
        	if(!Directory.Exists(logLocation)) {
        		Directory.CreateDirectory(logLocation);
        	}
			
			var testCaseIds = testCase.Parameters["TestCase"].Split(',').ToList().Select(i => i.Trim()).ToList();
        	
        	var result = 3; // this value (= untested) is NOT allowed!
        	
        	if(testRun.Status == ActivityStatus.Success)
        	{
        		result = 1; // Success
        	}
        	else if(testRun.Status == ActivityStatus.Failed)
        	{
        		result = 5; // Failed
        	}
        	
        	var commentBuilder = new StringBuilder("This test has been executed by Ranorex\n\n")
        		.Append("||| :Field               | :Value\n")
        		.Append("|| Run configuration: ".PadRight(25)+"| " + testSuite.SelectedRunConfig.Name+"\n");
        	
        	if(testRailTestRunId > 0) {
        		commentBuilder.Append("|| Test run: ".PadRight(25)+"| "+testRailTestRunName+"\n");
        	}
        	
        	commentBuilder
        		.Append("|| Test case: ".PadRight(25)+"| "+testCase.Name+"\n")
        		.Append("|| Test case coverage: ".PadRight(25)+"| "+String.Join(", ", testCaseIds)+"\n")
        		.Append("|| Parameters: ".PadRight(25)+"| ");
        	
        	var prefix = string.Empty;
	        foreach(var param in testCase.Parameters)
	        {
	        	commentBuilder.Append(prefix+param.Key+" = "+(param.Value == null ? "(null)" : param.Value)+"\n");
	        	
	        	prefix = "||".PadRight(25)+"| ";
	        }
	        
	        commentBuilder.Append("\n---\n\n");
	        
	        commentBuilder.Append("||| :Time stamp | :Report level | :Category            | :Message\n");
	        
	        var dataCount = 1;
	        foreach(var i in items)
	        {
	        	commentBuilder.Append("|| "+i.Timestamp+" | "+i.ReportLevel.Name.PadRight(13)+" | " + i.Category.PadRight(20) + " | " + i.Message);
	        	
	        	if(i.ImagePath != null)
	        	{
	        		var location = logLocation + "Data-"+dataCount+".png";
	        		File.Move(i.ImagePath, location);
	        		commentBuilder.Append("![Screenshot] ("+location+")");
	        		dataCount++;
	        	}
	        	else if(i.DataPath != null)
	        	{
	        		var location = logLocation + "Data "+dataCount+".txt";
	        		File.Move(i.DataPath, location);
	        		commentBuilder.Append("[Data] ("+location+")");
	        		dataCount++;
	        	}
	        	
	        	commentBuilder.Append("\n");
	        }
	        
	        var comment = commentBuilder.ToString();
        	
        	foreach(var testCaseId in testCaseIds)
        	{
        		var elapsed = (testRun.ElapsedTime.Milliseconds/1000);
        		
        		if(elapsed < 1) {
        			elapsed = 1;
        		}
        		
        		var data = new Dictionary<string, string>();
	        	data.Add("status_id", result.ToString());
	        	data.Add("elapsed", elapsed+" s");
	        	data.Add("comment", comment);
	        	//data.Add("custom_automation_log_file", "idk");//PrepareReportingForTestRun.PathToLogFile);
	        	
	        	var targetNode = testRailTestRunId > 0 ? "add_result_for_case/"+testRailTestRunId : "add_result";
	        	
	        	//var postResultResponse = Client.SendPost(targetNode + "/"+testCaseIds[0].Substring(1), data);
	        	
	        	/*Report.Info("Test result reported to TestRail:\n\n"+postResultResponse.ToString());*/
        	}
        	
        	File.WriteAllText(logLocation+"Report.txt", comment);
        	
			testCase = null;
			
			//Report.Info("END: TestRailReporter.End()");*/
		}
		
		public void LogText(Ranorex.ReportLevel level, string category, string message, bool escape, System.Collections.Generic.IDictionary<string, string> metaInfos)
		{
			if(message.Contains("Report_Test_Result_To_TestRail") || message.Contains("Prepare_Reporting_For_Test_Case"))
			{
				return;
			}
			
			var logEntry = new LogEntry {
				Timestamp = GetTimeStamp(),
				ReportLevel = level,
				Category = category,
				Message = message
			};
			
			items.Add(logEntry);
		}
		
		public void LogData(Ranorex.ReportLevel level, string category, string message, object data, System.Collections.Generic.IDictionary<string, string> metaInfos)
		{
			if(!Directory.Exists(@"U:\60_TestAutomation\tmp"))
			{
				Directory.CreateDirectory(@"U:\60_TestAutomation\tmp");
			}
			
			var logEntry = new LogEntry {
				Timestamp = GetTimeStamp(),
				ReportLevel = level,
				Category = category,
				Message = message
			};
			
			var location = @"U:\60_TestAutomation\tmp\"+Guid.NewGuid();
			
			if(data is Bitmap)
			{
				((Bitmap)data).Save(location, ImageFormat.Png);
				logEntry.ImagePath = location;
			} else {
				File.WriteAllText(location, data.ToString());
				logEntry.DataPath = location;
			}
			
			items.Add(logEntry);
			//dataCount++;
		}
		
		private string GetTimeStamp()
        {
			//return testRun.ElapsedTime.ToString("HH:mm:ss.fff", System.Globalization.CultureInfo.InvariantCulture);
            return System.DateTime.Now.ToString("HH:mm:ss.fff",
                                                System.Globalization.CultureInfo.InvariantCulture);
        }
	}
	
	class LogEntry
	{
		public String Timestamp {get;set;}
		public ReportLevel ReportLevel {get;set;}
		public string Category {get;set;}
		public string Message {get;set;}
		public string ImagePath {get;set;}
		public string DataPath {get;set;}
	}
}