Data Dumper

wiki

Here is one of the multiple ways to implement the serialization of a table. TableSerialization lists many other solutions. This version is written for Lua 5.1 and is quite long, because it tries to be as generic as possible and to keep its output string human readable. The drawback is that it is not the fastest one, mainly because it returns a string and does not write to a temporarily file. Still, there is a "fast" mode with fewer features, when execution time (or memory) is important. The function is copyrighted on MIT license, so is is free to use even in a commercial product.

DataDumper consists of a single Lua function, which could easily be put in a separate module or integrated into a bigger one. The function has four parameters, but only the first one is mandatory. It always returns a string value, which is valid Lua code. Simply executing this chunk will import back to a variable the complete structure of the original variable. For simple structures, there is only one Lua instruction like a table constructor, but some more complex features will output a script with more instructions.

All the following language features are supported:

Simple Lua types: nil, boolean, number, string
Tables are dumped recursively
Table metatables are also dumped recursively
Simple Lua functions (no upvalue) are dumped with loadstring
Lua closures with upvalues are also supported, using the debug library!
Known C functions are output using their original name
Complex tables structures with internal references are supported

What is not supported since it seems to be impossible, are these types:

Both light and full userdata
Coroutines
Other C functions

Here is the function declaration:

function DataDumper(value[, varname, fastmode, indent])

value can be of any supported type
varname: optional variable name. Depending on its form, the output will look like:
- nil: "return value"
- identifier: "varname = value"
- other: "varname".."value"
fastmode is a boolean value:
- true: optimizes for speed. Metatables, closures, C functions and references are not supported. Returns a code chunk without any space or new line!
- false: supports all advanced features and favors readable code with good indentation.
indent: the number of additional indentation level. Default is 0.

Examples

-- Define a shortcut function for testing

function dump(...)

  print(DataDumper(...), "\n---")

end



-- Simple types:

dump(8)         --> return 8

dump(true)      --> return true

dump()          --> return nil

dump('Hello')   --> return "Hello"



-- Functions:

f1 = print

function f2() return "Hello" end

do

  local val = 9

  function f3() return val end

end

dump(f1)        --> return print

dump(f2)        --> return loadstring("?LuaQ\000...

dump(f3)        --[[ the following 16 lines script:

local closures = {}

local function closure(t)

  closures[#closures+1] = t

  t[1] = assert(loadstring(t[1]))

  return t[1]

end

local t = closure {

  "?LuaQ\000...",

  9

}

for _,t in pairs(closures) do

  for i = 2,#t do

    debug.setupvalue(t[1], i-1, t[i])

  end

end]]



-- Tables:

dump({})        --> return {  }

dump({1,2,3})   --> return { 1, 2, 3 }

dump({a=9,b=2}) --> return { a=9, b=2 }

t1 = setmetatable({1},{2})

t2 = {}; t2.next = t2

t3 = {[false] = true, 'Hello', ["key1"] = 10, ["function"] = "keyword", {1,2}}

setmetatable(t3, {__index = t2})

t3[3] = t2

dump(t1)        --> return setmetatable({ 1 },{ 2 })

dump(t2)        --[[ the following 3 lines script:

local t = { next=nil }

t.next = t

return t

]]

dump(t3)        --[[ the following 14 lines script:

local t = setmetatable(

  {

    [false]=true,

    "Hello",

    { 1, 2 },

    { next=nil },

    ["function"] = "keyword",

    key1=10

  },

  { __index={ next=nil } }

)

getmetatable(t).__index.next = getmetatable(t).__index

t[3].next = t[3]

return t

]]



-- Parameters

dump(t1, 'a')   --> a = setmetatable({ 1 },{ 2 })

dump({}, '')    --> {  }

dump({ { {} } },a,true) --> return {{{},},}

Main code

Files:wiki_insecure/dumper.lua

--[[ DataDumper.lua

Copyright (c) 2007 Olivetti-Engineering SA



Permission is hereby granted, free of charge, to any person

obtaining a copy of this software and associated documentation

files (the "Software"), to deal in the Software without

restriction, including without limitation the rights to use,

copy, modify, merge, publish, distribute, sublicense, and/or sell

copies of the Software, and to permit persons to whom the

Software is furnished to do so, subject to the following

conditions:



The above copyright notice and this permission notice shall be

included in all copies or substantial portions of the Software.



THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES

OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT

HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,

WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

OTHER DEALINGS IN THE SOFTWARE.

]]



local dumplua_closure = [[

local closures = {}

local function closure(t) 

  closures[#closures+1] = t

  t[1] = assert(loadstring(t[1]))

  return t[1]

end



for _,t in pairs(closures) do

  for i = 2,#t do 

    debug.setupvalue(t[1], i-1, t[i]) 

  end 

end

]]



local lua_reserved_keywords = {

  'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 

  'function', 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 

  'return', 'then', 'true', 'until', 'while' }



local function keys(t)

  local res = {}

  local oktypes = { stringstring = true, numbernumber = true }

  local function cmpfct(a,b)

    if oktypes[type(a)..type(b)] then

      return a < b

    else

      return type(a) < type(b)

    end

  end

  for k in pairs(t) do

    res[#res+1] = k

  end

  table.sort(res, cmpfct)

  return res

end



local c_functions = {}

for _,lib in pairs{'_G', 'string', 'table', 'math', 

    'io', 'os', 'coroutine', 'package', 'debug'} do

  local t = _G[lib] or {}

  lib = lib .. "."

  if lib == "_G." then lib = "" end

  for k,v in pairs(t) do

    if type(v) == 'function' and not pcall(string.dump, v) then

      c_functions[v] = lib..k

    end

  end

end



function DataDumper(value, varname, fastmode, ident)

  local defined, dumplua = {}

  -- Local variables for speed optimization

  local string_format, type, string_dump, string_rep = 

        string.format, type, string.dump, string.rep

  local tostring, pairs, table_concat = 

        tostring, pairs, table.concat

  local keycache, strvalcache, out, closure_cnt = {}, {}, {}, 0

  setmetatable(strvalcache, {__index = function(t,value)

    local res = string_format('%q', value)

    t[value] = res

    return res

  end})

  local fcts = {

    string = function(value) return strvalcache[value] end,

    number = function(value) return value end,

    boolean = function(value) return tostring(value) end,

    ['nil'] = function(value) return 'nil' end,

    ['function'] = function(value) 

      return string_format("loadstring(%q)", string_dump(value)) 

    end,

    userdata = function() error("Cannot dump userdata") end,

    thread = function() error("Cannot dump threads") end,

  }

  local function test_defined(value, path)

    if defined[value] then

      if path:match("^getmetatable.*%)$") then

        out[#out+1] = string_format("s%s, %s)\n", path:sub(2,-2), defined[value])

      else

        out[#out+1] = path .. " = " .. defined[value] .. "\n"

      end

      return true

    end

    defined[value] = path

  end

  local function make_key(t, key)

    local s

    if type(key) == 'string' and key:match('^[_%a][_%w]*$') then

      s = key .. "="

    else

      s = "[" .. dumplua(key, 0) .. "]="

    end

    t[key] = s

    return s

  end

  for _,k in ipairs(lua_reserved_keywords) do

    keycache[k] = '["'..k..'"] = '

  end

  if fastmode then 

    fcts.table = function (value)

      -- Table value

      local numidx = 1

      out[#out+1] = "{"

      for key,val in pairs(value) do

        if key == numidx then

          numidx = numidx + 1

        else

          out[#out+1] = keycache[key]

        end

        local str = dumplua(val)

        out[#out+1] = str..","

      end

      if string.sub(out[#out], -1) == "," then

        out[#out] = string.sub(out[#out], 1, -2);

      end

      out[#out+1] = "}"

      return "" 

    end

  else 

    fcts.table = function (value, ident, path)

      if test_defined(value, path) then return "nil" end

      -- Table value

      local sep, str, numidx, totallen = " ", {}, 1, 0

      local meta, metastr = (debug or getfenv()).getmetatable(value)

      if meta then

        ident = ident + 1

        metastr = dumplua(meta, ident, "getmetatable("..path..")")

        totallen = totallen + #metastr + 16

      end

      for _,key in pairs(keys(value)) do

        local val = value[key]

        local s = ""

        local subpath = path

        if key == numidx then

          subpath = subpath .. "[" .. numidx .. "]"

          numidx = numidx + 1

        else

          s = keycache[key]

          if not s:match "^%[" then subpath = subpath .. "." end

          subpath = subpath .. s:gsub("%s*=%s*$","")

        end

        s = s .. dumplua(val, ident+1, subpath)

        str[#str+1] = s

        totallen = totallen + #s + 2

      end

      if totallen > 80 then

        sep = "\n" .. string_rep("  ", ident+1)

      end

      str = "{"..sep..table_concat(str, ","..sep).." "..sep:sub(1,-3).."}" 

      if meta then

        sep = sep:sub(1,-3)

        return "setmetatable("..sep..str..","..sep..metastr..sep:sub(1,-3)..")"

      end

      return str

    end

    fcts['function'] = function (value, ident, path)

      if test_defined(value, path) then return "nil" end

      if c_functions[value] then

        return c_functions[value]

      elseif debug == nil or debug.getupvalue(value, 1) == nil then

        return string_format("loadstring(%q)", string_dump(value))

      end

      closure_cnt = closure_cnt + 1

      local res = {string.dump(value)}

      for i = 1,math.huge do

        local name, v = debug.getupvalue(value,i)

        if name == nil then break end

        res[i+1] = v

      end

      return "closure " .. dumplua(res, ident, "closures["..closure_cnt.."]")

    end

  end

  function dumplua(value, ident, path)

    return fcts[type(value)](value, ident, path)

  end

  if varname == nil then

    varname = "return "

  elseif varname:match("^[%a_][%w_]*$") then

    varname = varname .. " = "

  end

  if fastmode then

    setmetatable(keycache, {__index = make_key })

    out[1] = varname

    table.insert(out,dumplua(value, 0))

    return table.concat(out)

  else

    setmetatable(keycache, {__index = make_key })

    local items = {}

    for i=1,10 do items[i] = '' end

    items[3] = dumplua(value, ident or 0, "t")

    if closure_cnt > 0 then

      items[1], items[6] = dumplua_closure:match("(.*\n)\n(.*)")

      out[#out+1] = ""

    end

    if #out > 0 then

      items[2], items[4] = "local t = ", "\n"

      items[5] = table.concat(out)

      items[7] = varname .. "t"

    else

      items[2] = varname

    end

    return table.concat(items)

  end

end

--PatrickRapin

Comment on using this: If you use this with large SWIG modules, it is PAINFULLY slow -- more than 1/2 second to load this module due to the huge number of c function calls. You can extract just the 'fastmode' and get most of the functionality without the huge startup cost. --grb

RecentChanges · preferences
edit · history
Last edited August 4, 2012 10:32 am GMT (diff)