Enhanced File Lines

wiki
Here are functions for portable line-by-line text processing (handling OS differences in '\r' and '\n' characters).
-- (c) 2008 David Manura. Licensed under the same terms as Lua (MIT).



-- file_lines(f) is similar to f:lines() for file f.

-- The main difference is that read_lines includes

-- new-line character sequences ("\n", "\r\n", "\r"),

-- if any, at the end of each line.  Embedded "\0" are also handled.



-- Caution: The newline behavior can depend on whether f is opened

-- in binary or ASCII mode.

--

local function file_lines(f)

  local CHUNK_SIZE = 1024

  local buffer = ""

  local pos_beg = 1

  return function()

    local pos, chars

    while 1 do

      pos, chars = buffer:match('()([\r\n].)', pos_beg)

      if pos or not f then

        break

      elseif f then

        local chunk = f:read(CHUNK_SIZE)

        if chunk then

          buffer = buffer:sub(pos_beg) .. chunk

          pos_beg = 1

        else

          f = nil

        end

      end

    end

    if not pos then

      pos = #buffer

    elseif chars == '\r\n' then

      pos = pos + 1

    end

    local line = buffer:sub(pos_beg, pos)

    pos_beg = pos + 1

    if #line > 0 then

      return line

    end    

  end

end





--

-- Splits string s into array of lines, returning the result.

-- New-line character sequences ("\n", "\r\n", "\r"),

-- if any, are included at the ends of the lines.

--

local function split_newlines(s)

  local ts = {}

  local posa = 1

  while 1 do

    local pos, chars = s:match('()([\r\n].?)', posa)

    if pos then

      if chars == '\r\n' then pos = pos + 1 end

      local line = s:sub(posa, pos)

      ts[#ts+1] = line

      posa = pos + 1

    else

      local line = s:sub(posa)

      if line ~= '' then ts[#ts+1] = line end

      break      

    end

  end

  return ts

end





--[=[slower implementation

local function split_newlines(s)

  local ts = {}

  local lastc

  s:gsub('([^\r\n]*)([\r\n])', function(a,b)

    if a == '' and lastc == '\r' and b == '\n' then

      ts[#ts] = ts[#ts] .. b

      lastc = nil

    else

      ts[#ts+1] = a .. b

      lastc = b

    end

    return ''

  end)

  local line = s:match('([^\r\n]+)$')

  if line then ts[#ts+1] = line end

  return ts

end

--]=]





-- test suite





-- utility function for test suite.

-- Create mock file for string s.

local function mock_file(s)

  local f = {}

  function f:read(n, ...)

    assert(type(n)=='number' and select('#', ...) == 0, 'NOT IMPL')

    local chunk = s:sub(1,n)

    s = s:sub(n+1)

    return chunk ~= '' and chunk or nil

  end

  return f

end





-- utility function for test suite.

local function mytostring(s)

  return type(s) == 'string'

         and string.format('%q', s):gsub('\n','n')

         or  tostring(s)

end





-- utility function for test suite.

local function asserteq(a,b,level)

  level = (level or 1) + 1

  if a ~= b then

    error(mytostring(a) .. '~=' .. mytostring(b), level)

  end

end





-- utility function for test suite (wrap file_lines)

local function wrap1(s)

  local f = mock_file(s)

  local ts = {}

  for line in file_lines(f) do ts[#ts+1] = line end

  return table.concat(ts, '|')

end





-- utility function for test suite (wrap split_newlines)

local function wrap2(s)

  return table.concat(split_newlines(s), '|')

end





local SZ = 1024 -- chunk size



-- test basics

for _,f in ipairs{wrap1, wrap2} do

for _,i in ipairs{0,1,2,SZ-3,SZ-2,SZ-1,SZ,SZ+1,SZ+2,SZ+3} do

  local s = (' '):rep(i)

  local function test(a, b)

    asserteq(f(s .. a), s .. b)

  end

  test('', '')

  test('\r', '\r')

  test('\n', '\n')

  test('a',  'a')

  test('\r\n', '\r\n')

  test('\n\r', '\n|\r')

  test('\r\r', '\r|\r')

  test('\n\n', '\n|\n')

  test('a\n',  'a\n')

  test('a\r',  'a\r')

  test('\na',  '\n|a')

  test('\ra',  '\r|a')

end end



-- check that two implementations are equivalent on a lot of data.

local cs = {'', 'a', '\r', '\n', ' '}

for _,i in ipairs{0,1,SZ-3,SZ-2,SZ-1,SZ,SZ+1,SZ+2,SZ+3} do

for j=0,1 do

  local s = (' '):rep(i + j * SZ)

  for _,c1 in ipairs(cs) do

  for _,c2 in ipairs(cs) do

  for _,c3 in ipairs(cs) do

  for _,c4 in ipairs(cs) do

  for _,c5 in ipairs(cs) do

    local s = c1 .. c2 .. c3 .. c4 ..c5

    local t1 = wrap1(s)

    local t2 = wrap2(s)

    asserteq(t1, t2)

  end end end end end

end end





print 'DONE'
Note: these functions are used in LuaPatch.
--DavidManura
Enhanced File Lines

See Also
