Benchmark Module

wiki

Now, here is my benchmarking module. I wrote it to find out, which is the fastest way to do some things in Lua. It requires "HiResTimer".

Benchmark.lua

---------------------------

--- Benchmark provides a set of functions, which measures and compares execution time of different code fragments

---------------------------



module("Benchmark",package.seeall)

local hrt=require"HiResTimer"





local bench_results={}

local total_diff=0.0



---------------------------

--- Call a function for several times and measure execution time.

-- @param name (string) literal to identify test in result table

-- @param func (function) function to be benchmarked

-- @param loops (number) how often this function should be called

----------------------------

function Bench(name,func,loops)

	loops=loops or 1000

	local q0=hrt.clock()

	for i=1,loops do

		func()

	end

	local q1=hrt.clock()

	local result=bench_results[name] or {count=0,diff=0}

	local diff=(q1-q0)

	result.count=result.count+loops

	result.diff=result.diff+diff

	total_diff=total_diff+diff

	bench_results[name]=result

end



---------------------------

--- Do Benchmark over a table of functions

-- @param functions (table) table of functions to check

-- @param loops (number) how often to call the function (optional, default 1000)

---------------------------

function BenchTable(functions,loops)

	loops=loops or 1000

	for name,func in pairs(functions) do

		Bench(name,func,loops)

	end

end



----------------------------

--- Printout benchmark results.

-- @param Output (function) to receive values (optional, default=io.write)

----------------------------

function Results(Output)



	--

	-- prepare the output

	--

	Output=Output or io.write

	local function printf(form,...)

		Output(string.format(form,...))

	end



	--

	-- calculate mean values

	-- create a table of names

	--

	local names={}

	local namlen=0

	for name,result in pairs(bench_results) do

		result.mean=result.diff/result.count

--~ 		printf("diff=%8.3g cnt=%d mean=%8.3g\n",result.diff,result.count,result.mean)

		names[#names+1]=name

		if #name>namlen then namlen=#name end

	end



	--

	-- sort table by mean value

	--

	local function comp(na,nb)

		return bench_results[na].mean<bench_results[nb].mean

	end

	table.sort(names,comp)



	--

	-- derive some reasonable output scaling

	--

	local max=bench_results[names[#names]].mean

	local fac,unit=1,"sec"

	if max<0.001 then

		fac,unit=1000000,"µs"

	elseif max<1.0 then

		fac,unit=1000,"ms"

	end



	--

	-- create a format string (due "%-*s" is missing in string.format)

	--

	local form=string.gsub("-- %-#s : %8.3f %s = %6d loops/s [%6.2f %%] %5.3f times faster\n","#",tostring(namlen))



	--

	-- now print the result

	--

	printf("-----------------------------------\n")

	printf("-- MAX = %8.3f %s\n",max*fac,unit)

	for i=1,#names do

		local name=names[i]

		result=bench_results[name]

		local ratio=result.mean*100/max

		local times=max/result.mean

		local loops=1/result.mean

		printf(form,name,result.mean*fac,unit,loops,ratio,times)

	end

	printf("-----------------------------------\n")

end



return Benchmark

Example : string concatenation

The question is, how to combine some literals and some variables to a string.

require"Benchmark"



local sf=string.format



local TestCases=

{



	TextConcat=function()

		local t,a,s='A','#',5

		local n=""

		for i=1,1000 do

			n="\symbol{circled"..s.."}="..t..a..i

		end

	end,



	-- extreme slow

	TableConcat=function()

		local t,a,s='A','#',5

		local n=""

		for i=1,1000 do

			n=table.concat{"\symbol{circled",s,"}=",t,a,i}

		end

	end,



	StringFormat=function ()

		local t,a,s='A','#',5

		local n=""

		for i=1,1000 do

			n=string.format("\symbol{circled%d}=%s%s%d",s,t,a,i)

		end

	end,



	FunctionLocalStringFormat=function()

		local t,a,s='A','#',5

		local n=""

		local sf=string.format

		for i=1,1000 do

			n=sf("\symbol{circled%d}=%s%s%d",s,t,a,i)

		end

	end,



	ModuleLocalStringFormat=function()

		local t,a,s='A','#',5

		local n=""

		for i=1,1000 do

			n=sf("\symbol{circled%d}=%s%s%d",s,t,a,i)

		end

	end

}

Benchmark.BenchTable(TestCases,100)

Benchmark.Results()

Result

-----------------------------------

-- MAX =    4.951 ms

-- ModuleLocalStringFormat   :    1.665 ms =    600 loops/s [ 33.63 %] 2.974 times faster

-- FunctionLocalStringFormat :    1.696 ms =    589 loops/s [ 34.26 %] 2.919 times faster

-- StringFormat              :    1.728 ms =    578 loops/s [ 34.90 %] 2.865 times faster

-- TextConcat                :    2.754 ms =    363 loops/s [ 55.64 %] 1.797 times faster

-- TableConcat               :    4.951 ms =    201 loops/s [100.00 %] 1.000 times faster

-----------------------------------

Example loop over array

-----------------------------------

-- setup test data

-----------------------------------



local array={}



for i=1,100 do

	local x={}

	for j=1,100 do

		x[j]=tostring(i)..tostring(j)

	end

	array[i]=x

end



local TestCases=

{

	for_i_array=function()

		local count=0

		for i=1,#array do

			local x=array[i]

			for j=1,#x do

				local y=x[j]

				-- do something with y

			end

		end

		return count

	end,



	for_ipairs=function()

		local count=0

		for i,x in ipairs(array) do

			for j,y in ipairs(x) do

				-- do something with y

			end

		end

		return count

	end

}



local Benchmark=require"Benchmark"

Benchmark.BenchTable(TestCases,1000)

Benchmark.Results()

Result

-----------------------------------

-- MAX =    1.127 ms

-- for_i_array :    0.689 ms =   1451 loops/s [ 61.12 %] 1.636 times faster

-- for_ipairs  :    1.127 ms =    887 loops/s [100.00 %] 1.000 times faster

-----------------------------------