Author Topic: Problem saving data into my ip log script  (Read 4266 times)

0 Members and 1 Guest are viewing this topic.

Offline GePP

  • Newbie
  • *
  • Posts: 7
  • Karma: +0/-0
Problem saving data into my ip log script
« on: 11 November, 2005, 23:02:51 »
I'm adding the log (ip and nick) to my bot but i have a problem: every time i save to file the hub stop to respond for some seconds.
Everything seems to work (logs are saved correctly) but i have got this strange situation... i cannot think it's normal but, since it's only few days i write lua code, i prefer to ask to experts :-)
I uses pto 0.3.3.21 and the script i paste you down here.
i tested it locally and also with external users, i also tested it on two different pc: one win2000 server and one winXP.

below the code (not yet finishd but i'll finish it after the solution of this problem)

Code: [Select]
GePPIPlog.lua
 
-- GePP IP LOG script ver 0.100
function NewUserConnected(user)
  FuncLogUser(user,"i")
end
function OpConnected(user)
  FuncLogUser(user,"i")
end
function UserDisconnected(user)
  FuncLogUser(user,"o")
end
function OpDisconnected(user)
  FuncLogUser(user,"o")
end
function Main()
  dofile("GePP/LogFunctions.lua")
  initlogvars()
  FunctLoadLog(os.date("*t", os.time()))
end
 
LogFunctions.lua
 
-- GePPipLOG ver 0.100
-- to merge this script into your one follow instructions on top of of every function
-- and also change the variable below to feet your need
loglocation = "gepp/iplog/"
-- call this function as first into both NewUserConnected and OpConnected putting "i" as inout parameter
-- also call this function as first into both UserDisconnected and OpDisconnected putting "o" as inout parameter
function FuncLogUser(user,inout)
  logdata = {}
  tmpdate = os.date("*t", os.time())
  logdata["day"]=tmpdate.day
  logdata["hour"]=tmpdate.hour
  logdata["min"]=tmpdate.min
  logdata["sec"]=tmpdate.sec
  logdata["nick"]=user.sName
  logdata["ip"]=user.sIP
  logdata["inout"]=inout
  if logindex ~= 0 and dinamiclog[logindex].day ~= logdata.day then
    flog = io.open(loglocation..tmpdate.year..tmpdate.month..dinamiclog[logindex].day..".lua", "w+")
    for i in dinamiclog do
      WriteEntry(flog,dinamiclog[i])
    end
--    flog:flush()
--    flog:close()
    initlogvars()
  end
  logindex = logindex + 1
  dinamiclog[logindex] = logdata
  if savecounter == 9 then
    savecounter = 0
--    flog = io.open(loglocation..tmpdate.year..tmpdate.month..tmpdate.day..".lua", "a+")
--    if not flog then
      flog = io.open(loglocation..tmpdate.year..tmpdate.month..tmpdate.day..".lua", "w+")
--    end
    for i=1,logindex do
      WriteEntry(flog,dinamiclog[i])
    end
--    flog:flush()
--    flog:close()
  else
    savecounter = savecounter + 1
  end
end
-- call this function in main giving this as parameter --> os.date("*t", os.time())
function FunctLoadLog(savinglog)
  flog = io.open(loglocation..savinglog.year..savinglog.month..savinglog.day..".lua")
  if flog then
    dofile(loglocation..savinglog.year..savinglog.month..savinglog.day..".lua")
    flog:close()
  end
end
-- internal script function: no need to call it into your script
function entry (singlelog)
  logindex = logindex + 1
  dinamiclog[logindex] = singlelog
end
-- internal script function: no need to call it into your script
function WriteEntry (flog,entrytowrite)
  flog:write("entry{day="..entrytowrite.day..",hour="..entrytowrite.hour..",min="..entrytowrite.min..",sec="..entrytowrite.sec..",nick=\""..entrytowrite.nick.."\",ip=\""..entrytowrite.ip.."\",inout=\""..entrytowrite.inout.."\"}\r\n")
end
-- call this function in main before to call FunctLoadLog
function initlogvars()
  logindex = 0
  dinamiclog = {}
  savecounter = 0
end

PtokaX forum

Problem saving data into my ip log script
« on: 11 November, 2005, 23:02:51 »

Offline GePP

  • Newbie
  • *
  • Posts: 7
  • Karma: +0/-0
(No subject)
« Reply #1 on: 11 November, 2005, 23:15:02 »
i hate smilies....
here is the code again (without smilies)

Code: [Select]
GePPIPlog.lua
 
-- GePP IP LOG script ver 0.100
function NewUserConnected(user)
  FuncLogUser(user,"i")
end
function OpConnected(user)
  FuncLogUser(user,"i")
end
function UserDisconnected(user)
  FuncLogUser(user,"o")
end
function OpDisconnected(user)
  FuncLogUser(user,"o")
end
function Main()
  dofile("GePP/LogFunctions.lua")
  initlogvars()
  FunctLoadLog(os.date("*t", os.time()))
end
 
LogFunctions.lua
 
-- GePPipLOG ver 0.100
-- to merge this script into your one follow instructions on top of of every function
-- and also change the variable below to feet your need
loglocation = "gepp/iplog/"
-- call this function as first into both NewUserConnected and OpConnected putting "i" as inout parameter
-- also call this function as first into both UserDisconnected and OpDisconnected putting "o" as inout parameter
function FuncLogUser(user,inout)
  logdata = {}
  tmpdate = os.date("*t", os.time())
  logdata["day"]=tmpdate.day
  logdata["hour"]=tmpdate.hour
  logdata["min"]=tmpdate.min
  logdata["sec"]=tmpdate.sec
  logdata["nick"]=user.sName
  logdata["ip"]=user.sIP
  logdata["inout"]=inout
  if logindex ~= 0 and dinamiclog[logindex].day ~= logdata.day then
    flog = io.open(loglocation..tmpdate.year..tmpdate.month..dinamiclog[logindex].day..".lua", "w+")
    for i in dinamiclog do
      WriteEntry(flog,dinamiclog[i])
    end
--    flog:flush()
--    flog:close()
    initlogvars()
  end
  logindex = logindex + 1
  dinamiclog[logindex] = logdata
  if savecounter == 9 then
    savecounter = 0
--    flog = io.open(loglocation..tmpdate.year..tmpdate.month..tmpdate.day..".lua", "a+")
--    if not flog then
      flog = io.open(loglocation..tmpdate.year..tmpdate.month..tmpdate.day..".lua", "w+")
--    end
    for i=1,logindex do
      WriteEntry(flog,dinamiclog[i])
    end
--    flog:flush()
--    flog:close()
  else
    savecounter = savecounter + 1
  end
end
-- call this function in main giving this as parameter --> os.date("*t", os.time())
function FunctLoadLog(savinglog)
  flog = io.open(loglocation..savinglog.year..savinglog.month..savinglog.day..".lua")
  if flog then
    dofile(loglocation..savinglog.year..savinglog.month..savinglog.day..".lua")
    flog:close()
  end
end
-- internal script function: no need to call it into your script
function entry (singlelog)
  logindex = logindex + 1
  dinamiclog[logindex] = singlelog
end
-- internal script function: no need to call it into your script
function WriteEntry (flog,entrytowrite)
  flog:write("entry{day="..entrytowrite.day..",hour="..entrytowrite.hour..",min="..entrytowrite.min..",sec="..entrytowrite.sec..",nick=\""..entrytowrite.nick.."\",ip=\""..entrytowrite.ip.."\",inout=\""..entrytowrite.inout.."\"}\r\n")
end
-- call this function in main before to call FunctLoadLog
function initlogvars()
  logindex = 0
  dinamiclog = {}
  savecounter = 0
end

Offline DJ Bert

  • Triple Ace
  • **
  • Posts: 211
  • Karma: +6/-1
  • Guardian Scripter
(No subject)
« Reply #2 on: 11 November, 2005, 23:40:40 »
It seems to be no errors.

1. how many users are connected?
2. what is your connection (down/up)?
3. how many scripts are loaded?

It's possible to move some scripts up or down to work perfectly

Offline GePP

  • Newbie
  • *
  • Posts: 7
  • Karma: +0/-0
(No subject)
« Reply #3 on: 12 November, 2005, 07:48:36 »
one test was done with only me connected locally, another one still only me but connecting from another pc inside my lan and last test also with another user external to my lan.
As you see i setted up the saving after 10 logs just to be able to test so it's easy to reach the saving point.

i have adsl 1200/512, normally (and also during the tests) dw bandwith is free and up bandwith is half occupied by one of my DChubs (userlimit 270, during tests less then 200 user connected)

first time there was only my bot running: it' composed by one main script (built in there is this iplog script) and two small external script. the main script is the first in execution list.
second time i used only the iplog script i pasted here.

about first question: if i have problems doung test with one or two user what about when i'll use it with 4000/5000 user? i really hope it's not a matter of user or else i'll never use it!

Offline GePP

  • Newbie
  • *
  • Posts: 7
  • Karma: +0/-0
(No subject)
« Reply #4 on: 12 November, 2005, 08:07:58 »
one more intresting explanation:

all users connected during the "critical time" of this script still continue to be able to post in main chat (and also to serch e download i suppose, but not tested).
what all users cannot do durig that period is to exit and entering the hub: they connect and immediatly disconnect from hub! (also a nwe user find this situation: connect and disconnect)

Offline Dessamator

  • Scripter
  • Emperor
  • ******
  • Posts: 1 262
  • Karma: +13/-9
(No subject)
« Reply #5 on: 12 November, 2005, 20:26:23 »
Code: [Select]
tmpdate = os.date("*t", os.time())
  logdata["day"]=tmpdate.day
  logdata["hour"]=tmpdate.hour
  logdata["min"]=tmpdate.min
  logdata["sec"]=tmpdate.sec


I dont understand why u are saving so many small things, why not just save the os.date("%X") or something, besides that, its probably using up a lot of resources because ur table is too large, rather than saving it piece by piece you save it when its huge, so until the table is saved the hub will be stuck.

My suggestion, cut down on useless things in ur table. and dont save the table when it gets too huge. Besides that ur saving each nick that logs in or out, with a very busy hub, that log can get huge in secs, check out the statlogger 1.0a for some hints.
Ignorance is Bliss.

Offline bastya_elvtars

  • Forum God
  • ****
  • Posts: 3 752
  • Karma: +173/-7
  • The rock n' roll doctor
    • The FreshStuff3 Site
(No subject)
« Reply #6 on: 14 November, 2005, 13:34:40 »
For logging, I would use a+ mode for files and saving entries one by one. Still better than saving things periodically. Same entries can easily be filtered (thus the file gets compacted) on startup if necessary.
Everything could have been anything else and it would have just as much meaning.

Offline GePP

  • Newbie
  • *
  • Posts: 7
  • Karma: +0/-0
(No subject)
« Reply #7 on: 14 November, 2005, 13:56:57 »
saving one by one could be too much for hubs with about 1 user per second!

my firs idea was to save, for example, 10 by 10 but now i changed idea.
Thanks to good ideas seen here and also from plop i decided to save on timer (and of course on exit). Doing like this i can set it up a correct time for every kind of hub. the bigger is the long is the time interval.

About the error i realize it's not a matter of my script but it's something related to security issues: after some in/out of user with same ip the hubsoft stop to respond. I tested it without script and the problem still remain so it's like i say!

Offline GePP

  • Newbie
  • *
  • Posts: 7
  • Karma: +0/-0
(No subject)
« Reply #8 on: 16 November, 2005, 23:32:14 »
I've done it and now it works!

i tested it on my hub (350 user minimum during night and about 700 maximum during day) and soon i'll test it on one 4000 user hub.
On 700 user hub it uses 10mb of ram and the daily log on disk takes about 900 kb.

Thanks for suggestion to you all and also to plop.

(i suppose since now it's a finishd script the place for it it's not here but since i'm not a forums expert i start to put it here)

Code: [Select]
-- (save this as GePPIPlogMain.lua and put it into script directory)
-- GePPIPlogMain script bot ver 0.3
-- by GePP using some lines of one plop script and some other ideas from other scripts
-- Commands: log, savesettings
-- Functions: IP/nick log (log command only work with nick for now)

-- small "how to use": setup permission of use for log command manually changing the scrip
--                     unpack this bot file into script directory and activate it into pto


iBotName = frmHub:GetHubBotName()

function NewUserConnected(user)
  FuncLogUser(user,"i")
end

function OpConnected(user)
  FuncLogUser(user,"i")
end

function UserDisconnected(user)
  FuncLogUser(user,"o")
end

function OpDisconnected(user)
  FuncLogUser(user,"o")
end

function OnExit()
  SaveLog()
end

function OnTimer()
  SaveLog()
end

function nopipeindata(data)
  local s,e,newdata = string.find(data, "(.*)|")
  return newdata
end

function ProcessChatArrival(user, data)
  data = nopipeindata(data)
  if user.bOperator then
    local s,e,sCom, iVal = string.find(data, "%b<>%s+(%S+)%s*(%S*).*")
    if sCom then
      if sCom == "!log" then
        if iVal == "" then
          SendPmToNick(user.sName,iBotName, "usage: !log to get log info of ")
        else
          local sPace = string.char(32)..string.char(32)..string.char(32)..string.char(32)
          local info = "\r\n\r\n\t\r\n"
          info = info.."\t"..sPace.."Log for "..iVal.." : ".."\r\n"
          info = info.."\t"..sPace.."*************".."\r\n"
          for i in dinamiclog do
            if dinamiclog[i].n == iVal then
              info = info.."\t"..sPace.."On : "..(os.date("%c", dinamiclog[i].t)).." user : "..iVal.." whent in or out (i/o) -->"..dinamiclog[i].o.." using this ip : "..dinamiclog[i].i.."\r\n"
            end
          end
          info = info.."\t"..sPace.."*************".."\r\n"
          info = info.."\t"..sPace.."End of log for "..iVal.."\r\n"
          SendPmToNick(user.sName,iBotName, info)
        end
        return 1
      elseif sCom == "!savesettings" or sCom == "!ss" then
        SaveLog()
        SendPmToNick(user.sName,iBotName, "Logs successfully saved")
        return 1
      end
    end
  end
end

function ToArrival(user, data)
  local s,e,to = string.find(data, "$To: (%S+%s?%S*)%sFrom:")
  if to == iBotName then
    ProcessChatArrival(user, data)
  end
  return 0
end

function ChatArrival(user, data)
  retchat = ProcessChatArrival(user, data)
  return retchat
end

function Main()
  dofile("GePP/LogFunctions.lua")
  initlogvars()
  FunctLoadLog(os.date("*t", os.time()))
end

-- (save this as LogFunctions.lua and put it into script/GePP directory

-- GePPipLOG ver 0.2
-- to merge this script into your one follow instructions on top of of every function
-- and also change the variable below to feet your need
-- changes respect to ver 0.1
-- bug fixed: during changing day old day last logs now are saved
-- added mkdir to create log directory if missing

loglocation = "gepp/IPlog/"    -- at least one directory: you cannot save log into main script directory: for example put here "IPlog"
SavingTime = 60                -- time in minutes between one automatic save to disk and next one

-- call this function as first into both NewUserConnected and OpConnected putting "i" as inout parameter
-- also call this function as first into both UserDisconnected and OpDisconnected putting "o" as inout parameter
function FuncLogUser(user,inout)
  local logdata = {}
  logdata["t"]=os.time()
  logdata["n"]=user.sName
  logdata["i"]=user.sIP
  logdata["o"]=inout
  logindex = logindex + 1
  dinamiclog[logindex] = logdata
end

-- call this function ontimer and onexit
function SaveLog()
  if not (lastsaved == 0 and logindex == 0) then
    local tmpdate = os.date("*t", os.time())
    if lastsaved == 0 then
      if os.date("*t", dinamiclog[logindex].t).day == os.date("*t", dinamiclog[1].t).day then
        local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[logindex].t).day..".lua", "w+")
        if not flog then
          os.execute("mkdir .\\"..string.gsub(loglocation, "/", "\\"))
          local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[logindex].t).day..".lua", "w+")
        end
        for i in dinamiclog do
          WriteEntry(flog,dinamiclog[i])
        end
        lastsaved = logindex
        flog:flush()
        flog:close()
      else
        for i in dinamiclog do
          if i == 1 then
            local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[1].t).day..".lua", "w+")
            if not flog then
              os.execute("mkdir .\\"..string.gsub(loglocation, "/", "\\"))
              local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[1].t).day..".lua", "w+")
            end
          elseif os.date("*t", dinamiclog[i-1].t).day ~= os.date("*t", dinamiclog[i].t).day then
            local newfirst = i
            flog:flush()
            flog:close()            
            local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[1].t).day..".lua", "w+")
            if not flog then
              os.execute("mkdir .\\"..string.gsub(loglocation, "/", "\\"))
              local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[1].t).day..".lua", "w+")
            end
          end
          WriteEntry(flog,dinamiclog[i])
          local lasti = i
        end
        tempdinamiclog = {}
        for i = newfirst,lasti do
          tempdinamiclog[i-newfirst+1]=dinamiclog[i]
        end
        dinamiclog = {}
        dinamiclog = tempdinamiclog
        logindex = lasti - newfirst + 1
        lastsaved = logindex
        tempdinamiclog = nil
      end
    else
      if logindex > lastsaved then
        tempdinamiclog = {}
        local i = lastsaved+1
        repeat
          tempdinamiclog[i-lastsaved]=dinamiclog[i]
          i = i + 1
        until dinamiclog[i] == nil
        if os.date("*t", dinamiclog[logindex].t).day == os.date("*t", dinamiclog[lastsaved+1].t).day then
          local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", tempdinamiclog[1].t).day..".lua", "a+")
          for i in tempdinamiclog do
            WriteEntry(flog,tempdinamiclog[i])
          end      
          lastsaved = logindex
        else
          for i in tempdinamiclog do
            if i == 1 then
              local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", tempdinamiclog[1].t).day..".lua", "a+")
            elseif tempdinamiclog[i-1].day ~= tempdinamiclog[i].day then
              newfirst = i
              flog:flush()
              flog:close()            
              local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", tempdinamiclog[1].t).day..".lua", "a+")
            end
            WriteEntry(flog,tempdinamiclog[i])
            lasti = i
          end
          dinamiclog = {}
          dinamiclog = tempdinamiclog
          logindex = lasti
          lastsaved = logindex
          tempdinamiclog = nil      
        end
      end
    end
  end
end

-- call this function in main giving this as parameter --> os.date("*t", os.time())
function FunctLoadLog(savinglog)
  flog = io.open(loglocation..savinglog.year..savinglog.month..savinglog.day..".lua")
  if flog then
    dofile(loglocation..savinglog.year..savinglog.month..savinglog.day..".lua")
    flog:close()
  end
end

-- internal script function: no need to call it into your script
function e (singlelog)
  logindex = logindex + 1
  lastsaved = lastsaved + 1
  dinamiclog[logindex] = singlelog
end

-- internal script function: no need to call it into your script
function WriteEntry (flog,entrytowrite)
-- originario  flog:write("entry{day="..entrytowrite.day..",hour="..entrytowrite.hour..",min="..entrytowrite.min..",sec="..entrytowrite.sec..",nick=\""..entrytowrite.nick.."\",ip=\""..entrytowrite.ip.."\",inout=\""..entrytowrite.inout.."\"}\r\n")
  flog:write("e{t="..entrytowrite.t..",n=\""..entrytowrite.n.."\",i=\""..entrytowrite.i.."\",o=\""..entrytowrite.o.."\"}\r\n")
end

-- call this function in main before to call FunctLoadLog
function initlogvars()
  logindex = 0
  lastsaved = 0
  dinamiclog = {}
  savecounter = 0
  SetTimer(SavingTime*60000) -- Timer for saving logs on given interval (fist number in minutes)
  StartTimer()
end

Offline bastya_elvtars

  • Forum God
  • ****
  • Posts: 3 752
  • Karma: +173/-7
  • The rock n' roll doctor
    • The FreshStuff3 Site
(No subject)
« Reply #9 on: 16 November, 2005, 23:58:33 »
You can always upload it here.
Everything could have been anything else and it would have just as much meaning.

Offline GePP

  • Newbie
  • *
  • Posts: 7
  • Karma: +0/-0
(No subject)
« Reply #10 on: 19 November, 2005, 16:04:45 »
I realize there was a bug: down here i post the function SaveLog bugfixed.

Now it works and i tested it on one 700 user hub and also on a 4000 user hub: It uses about 30Mb on the 4000 user hub and about 10Mb on the 700 one.

Code: [Select]
function SaveLog()
  if not (lastsaved == 0 and logindex == 0) then
    local tmpdate = os.date("*t", os.time())
    if lastsaved == 0 then
      if os.date("*t", dinamiclog[logindex].t).day == os.date("*t", dinamiclog[1].t).day then
        local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[logindex].t).day..".lua", "w+")
        if not flog then
          os.execute("mkdir .\\"..string.gsub(loglocation, "/", "\\"))
          local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[logindex].t).day..".lua", "w+")
        end
        for i in dinamiclog do
          WriteEntry(flog,dinamiclog[i])
        end
        lastsaved = logindex
        flog:flush()
        flog:close()
      else
        local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[1].t).day..".lua", "w+")
        if not flog then
          os.execute("mkdir .\\"..string.gsub(loglocation, "/", "\\"))
          local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", dinamiclog[1].t).day..".lua", "w+")
        end
        i = 0
        repeat
          i = i + 1
          WriteEntry(flog,dinamiclog[i])
        until os.date("*t", dinamiclog[i+1].t).day ~= os.date("*t", dinamiclog[i].t).day
        flog:flush()
        flog:close()            
        tempdinamiclog = {}
        logindex = 0
        repeat
          i = i + 1
          logindex = logindex + 1
          tempdinamiclog[logindex]= dinamiclog[i]
        until dinamiclog[i+1] == nil
        lastsaved = 0
        dinamiclog = tempdinamiclog
        tempdinamiclog = nil      
      end
    else
      if logindex > lastsaved then
        tempdinamiclog = {}
        local i = lastsaved+1
        repeat
          tempdinamiclog[i-lastsaved]=dinamiclog[i]
          i = i + 1
        until dinamiclog[i] == nil
        lasti = i-1
        if os.date("*t", tempdinamiclog[1].t).day == os.date("*t", dinamiclog[logindex].t).day then
          local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", tempdinamiclog[1].t).day..".lua", "a+")
          for i in tempdinamiclog do
            WriteEntry(flog,tempdinamiclog[i])
          end      
          if os.date("*t", dinamiclog[lastsaved].t).day == os.date("*t", dinamiclog[logindex].t).day then
            lastsaved = logindex
          else
            dinamiclog = {}
            dinamiclog = tempdinamiclog
            logindex = logindex - lastsaved
            lastsaved = logindex
          end
        else
          local flog = io.open(loglocation..tmpdate.year..tmpdate.month..os.date("*t", tempdinamiclog[1].t).day..".lua", "a+")
          i = 0
          repeat
            i = i + 1
            WriteEntry(flog,tempdinamiclog[i])
          until os.date("*t", tempdinamiclog[i+1].t).day ~= os.date("*t", tempdinamiclog[i].t).day
          flog:flush()
          flog:close()            
          dinamiclog = {}
          logindex = 0
          repeat
            i = i + 1
            logindex = logindex + 1
            dinamiclog[logindex]= tempdinamiclog[i]
          until tempdinamiclog[i+1] == nil
          lastsaved = 0
          tempdinamiclog = nil      
        end
      end
    end
  end
end

PtokaX forum

(No subject)
« Reply #10 on: 19 November, 2005, 16:04:45 »