Another update from ingydotnet^djgoku
This commit is contained in:
Ingy döt Net
parent
91df62d461
commit
948b86eafa
7604 changed files with 108452 additions and 22726 deletions
|
|
@ -1,7 +1,7 @@
|
|||
{{Template:clarify task}}
|
||||
Often data is produced by one program, in the wrong format for later use by another program or person. In these situations another program can be written to parse and transform the original data into a format useful to the other. The term "Data Munging" is [http://www.google.co.uk/search?q=%22data+munging%22 often] used in programming circles for this task.
|
||||
|
||||
A [http://groups.google.co.uk/group/comp.lang.awk/msg/0ecba3a3fbf247d8?hl=en request] on the comp.lang.awk newsgroup lead to a typical data munging task:
|
||||
A [http://groups.google.co.uk/group/comp.lang.awk/msg/0ecba3a3fbf247d8?hl=en request] on the comp.lang.awk newsgroup led to a typical data munging task:
|
||||
<pre>I have to analyse data files that have the following format:
|
||||
Each row corresponds to 1 day and the field logic is: $1 is the date,
|
||||
followed by 24 value/flag pairs, representing measurements at 01:00,
|
||||
|
|
|
|||
119
Task/Text-processing-1/Eiffel/text-processing-1.e
Normal file
119
Task/Text-processing-1/Eiffel/text-processing-1.e
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
class
|
||||
APPLICATION
|
||||
|
||||
create
|
||||
make
|
||||
|
||||
feature
|
||||
|
||||
make
|
||||
-- Summary statistics for 'hash'.
|
||||
local
|
||||
reject, accept, reading_total: INTEGER
|
||||
total, average, file_total: REAL
|
||||
do
|
||||
read_wordlist
|
||||
across
|
||||
hash as h
|
||||
loop
|
||||
io.put_string (h.key + "%T")
|
||||
reject := 0
|
||||
accept := 0
|
||||
total := 0
|
||||
across
|
||||
h.item as data
|
||||
loop
|
||||
if data.item.flag > 0 then
|
||||
accept := accept + 1
|
||||
total := total + data.item.val
|
||||
else
|
||||
reject := reject + 1
|
||||
end
|
||||
end
|
||||
file_total := file_total + total
|
||||
reading_total := reading_total + accept
|
||||
io.put_string ("accept: " + accept.out + "%Treject: " + reject.out + "%Ttotal: " + total.out + "%T")
|
||||
average := total / accept.to_real
|
||||
io.put_string ("average: " + average.out + "%N")
|
||||
end
|
||||
io.put_string ("File total: " + file_total.out + "%N")
|
||||
io.put_string ("Readings total: " + reading_total.out + "%N")
|
||||
find_longest_gap
|
||||
end
|
||||
|
||||
find_longest_gap
|
||||
-- Longest gap (flag values <= 0).
|
||||
local
|
||||
count: INTEGER
|
||||
longest_gap: INTEGER
|
||||
end_date: STRING
|
||||
do
|
||||
create end_date.make_empty
|
||||
across
|
||||
hash as h
|
||||
loop
|
||||
across
|
||||
h.item as data
|
||||
loop
|
||||
if data.item.flag <= 0 then
|
||||
count := count + 1
|
||||
else
|
||||
if count > longest_gap then
|
||||
longest_gap := count
|
||||
end_date := h.key
|
||||
end
|
||||
count := 0
|
||||
end
|
||||
end
|
||||
end
|
||||
io.put_string ("%NThe longest gap is " + longest_gap.out + ". It ends at the date stamp " + end_date + ". %N")
|
||||
end
|
||||
|
||||
original_list: STRING = "readings.txt"
|
||||
|
||||
read_wordlist
|
||||
-- Preprocessed wordlist in 'hash'.
|
||||
local
|
||||
l_file: PLAIN_TEXT_FILE
|
||||
data: LIST [STRING]
|
||||
by_dates: LIST [STRING]
|
||||
date: STRING
|
||||
data_tup: TUPLE [val: REAL; flag: INTEGER]
|
||||
data_arr: ARRAY [TUPLE [val: REAL; flag: INTEGER]]
|
||||
i: INTEGER
|
||||
do
|
||||
create l_file.make_open_read_write (original_list)
|
||||
l_file.read_stream (l_file.count)
|
||||
data := l_file.last_string.split ('%N')
|
||||
l_file.close
|
||||
create hash.make (data.count)
|
||||
across
|
||||
data as d
|
||||
loop
|
||||
if not d.item.is_empty then
|
||||
by_dates := d.item.split ('%T')
|
||||
date := by_dates [1]
|
||||
by_dates.prune (date)
|
||||
create data_tup
|
||||
create data_arr.make_empty
|
||||
from
|
||||
i := 1
|
||||
until
|
||||
i > by_dates.count - 1
|
||||
loop
|
||||
data_tup := [by_dates [i].to_real, by_dates [i + 1].to_integer]
|
||||
data_arr.force (data_tup, data_arr.count + 1)
|
||||
i := i + 2
|
||||
end
|
||||
hash.put (data_arr, date)
|
||||
if not hash.inserted then
|
||||
date.append ("_double_date_stamp")
|
||||
hash.put (data_arr, date)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
hash: HASH_TABLE [ARRAY [TUPLE [val: REAL; flag: INTEGER]], STRING]
|
||||
|
||||
end
|
||||
92
Task/Text-processing-1/Fortran/text-processing-1.f
Normal file
92
Task/Text-processing-1/Fortran/text-processing-1.f
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
Crunches a set of hourly data. Starts with a date, then 24 pairs of value,indicator for that day, on one line.
|
||||
INTEGER Y,M,D !Year, month, and day.
|
||||
INTEGER GOOD(24) !The indicators.
|
||||
REAL*8 V(24),VTOT,T !The grist.
|
||||
INTEGER NV,N,NB !Number of good values overall, and in a day.
|
||||
INTEGER I,NREC,HIC !Some counters.
|
||||
INTEGER BI,BN,BBI,BBN !Stuff to locate the longest run of bad data,
|
||||
CHARACTER*10 BDATE,BBDATE !Along with the starting date.
|
||||
LOGICAL INGOOD !State flipper for the runs of data.
|
||||
INTEGER IN,MSG !I/O mnemonics.
|
||||
CHARACTER*666 ACARD !Scratchpad, of sufficient length for all expectation.
|
||||
IN = 10 !Unit number for the input file.
|
||||
MSG = 6 !Output.
|
||||
OPEN (IN,FILE="Readings1.txt", FORM="FORMATTED", !This should be a function.
|
||||
1 STATUS ="OLD",ACTION="READ") !Returning success, or failure.
|
||||
NB = 0 !No bad values read.
|
||||
NV = 0 !Nor good values read.
|
||||
VTOT = 0 !Their average is to come.
|
||||
NREC = 0 !No records read.
|
||||
HIC = 0 !Provoking no complaints.
|
||||
INGOOD = .TRUE. !I start in hope.
|
||||
BBN = 0 !And the longest previous bad run is short.
|
||||
Chew into the file.
|
||||
10 READ (IN,11,END=100,ERR=666) L,ACARD(1:MIN(L,LEN(ACARD))) !With some protection.
|
||||
NREC = NREC + 1 !So, a record has been read.
|
||||
11 FORMAT (Q,A) !Obviously, Q ascertains the length of the record being read.
|
||||
READ (ACARD,12,END=600,ERR=601) Y,M,D !The date part is trouble, as always.
|
||||
12 FORMAT (I4,2(1X,I2)) !Because there are no delimiters between the parts.
|
||||
READ (ACARD(11:L),*,END=600,ERR=601) (V(I),GOOD(I),I = 1,24) !But after the date, delimiters abound.
|
||||
Calculations. Could use COUNT(array) and SUM(array), but each requires its own pass through the array.
|
||||
20 T = 0 !Start on the day's statistics.
|
||||
N = 0 !No values yet.
|
||||
DO I = 1,24 !So, scan the cargo and do all the twiddling in one pass..
|
||||
IF (GOOD(I).GT.0) THEN !A good value?
|
||||
N = N + 1 !Yes. Count it in.
|
||||
T = T + V(I) !And augment for the average.
|
||||
IF (.NOT.INGOOD) THEN !Had we been ungood?
|
||||
INGOOD = .TRUE. !Yes. But now it changes.
|
||||
IF (BN.GT.BBN) THEN !The run just ending: is it longer?
|
||||
BBN = BN !Yes. Make it the new baddest.
|
||||
BBI = BI !Recalling its start index,
|
||||
BBDATE = BDATE !And its start date.
|
||||
END IF !So much for bigger badness.
|
||||
END IF !Now we're in good data.
|
||||
ELSE !Otherwise, a bad value is upon us.
|
||||
IF (INGOOD) THEN !Were we good?
|
||||
INGOOD = .FALSE. !No longer. A new bad run is starting.
|
||||
BDATE = ACARD(1:10) !Recall the date for this starter.
|
||||
BI = I !And its index.
|
||||
BN = 0 !Start the run-length counter.
|
||||
END IF !So much for a fall.
|
||||
BN = BN + 1 !Count another bad value.
|
||||
END IF !Good or bad, so much for that value.
|
||||
END DO !On to the next.
|
||||
Commentary for the day's data..
|
||||
IF (N.LE.0) THEN !I prefer to avoid dividing by zero.
|
||||
WRITE (MSG,21) NREC,ACARD(1:10) !So, no average to report.
|
||||
21 FORMAT ("Record",I8," (",A,") has no good data!") !Just a remark.
|
||||
ELSE !But otherwise,
|
||||
WRITE(MSG,22) NREC,ACARD(1:10),N,T/N !An average is possible.
|
||||
22 FORMAT("Record",I8," (",A,")",I3," good, average",F9.3) !So here it is.
|
||||
NB = NB + 24 - N !Count the bad by implication.
|
||||
NV = NV + N !Count the good directly.
|
||||
VTOT = VTOT + T !Should really sum deviations from a working average.
|
||||
END IF !So much for that line.
|
||||
GO TO 10 !More! More! I want more!!
|
||||
|
||||
Complaints. Should really distinguish between trouble in the date part and in the data part.
|
||||
600 WRITE (MSG,*) '"END" declared - insufficient data?' !Not enough numbers, presumably.
|
||||
GO TO 602 !Reveal the record.
|
||||
601 WRITE (MSG,*) '"ERR" declared - improper number format?' !Ah, but which number?
|
||||
602 WRITE (MSG,603) NREC,L,ACARD(1:L) !Anyway, reveal the uninterpreted record.
|
||||
603 FORMAT(" Record ",I0,", length ",I0," reads ",A) !Just so.
|
||||
HIC = HIC + 1 !This may grow into a habit.
|
||||
IF (HIC.LE.12) GO TO 10 !But if not yet, try the next record.
|
||||
STOP "Enough distaste." !Or, give up.
|
||||
666 WRITE (MSG,101) NREC,"format error!" !For A-style data? Should never happen!
|
||||
GO TO 900 !But if it does, give up!
|
||||
|
||||
Closedown.
|
||||
100 WRITE (MSG,101) NREC,"then end-of-file" !Discovered on the next attempt.
|
||||
101 FORMAT (" Record ",I0,": ",A) !A record number plus a remark.
|
||||
WRITE (MSG,102) NV,NB,VTOT/NV !The overall results.
|
||||
102 FORMAT (I8," values, ",I0," bad. Average",F9.4) !This should do.
|
||||
IF (BBN.LE.0) THEN !Now for a special report.
|
||||
WRITE (MSG,*) "No bad value presented, so no longest run." !Unneeded!
|
||||
ELSE !But actually, the example data has some bad values.
|
||||
WRITE (MSG,103) BBN,BBI,BBDATE !And this is for the longest encountered.
|
||||
103 FORMAT ("Longest bad run: ",I0,", starting hour ",I0," on ",A) !Just so.
|
||||
END IF !Enough remarks.
|
||||
900 CLOSE(IN) !Done.
|
||||
END !Spaghetti rules.
|
||||
|
|
@ -1,72 +1,65 @@
|
|||
/*REXX program to process instrument data from a data file. */
|
||||
numeric digits 20 /*allow for bigger numbers. */
|
||||
ifid='READINGS.TXT' /*the input file. */
|
||||
ofid='READINGS.OUT' /*the outut file. */
|
||||
grandSum=0 /*grand sum of whole file. */
|
||||
grandflg=0 /*grand num of flagged data. */
|
||||
/*REXX program to process instrument data from a data file. */
|
||||
numeric digits 20 /*allow for bigger (precision) numbers.*/
|
||||
ifid='READINGS.TXT' /*the name of the input file. */
|
||||
ofid='READINGS.OUT' /* " " " " output " */
|
||||
grandSum=0 /*the grand sum of whole file. */
|
||||
grandFlg=0 /*the grand number of flagged data. */
|
||||
grandOKs=0
|
||||
longFlag=0 /*longest period of flagged data.*/
|
||||
contFlag=0 /*longest continous flagged data.*/
|
||||
w=16 /*width of fields when displayed.*/
|
||||
Lflag=0 /*the longest period of flagged data. */
|
||||
Cflag=0 /*the longest continous flagged data. */
|
||||
w=16 /*the width of fields when displayed. */
|
||||
|
||||
do recs=1 while lines(ifid)\==0 /*read until finished. */
|
||||
rec=linein(ifid) /*read the next record (line). */
|
||||
parse var rec datestamp Idata /*pick off the dateStamp & data. */
|
||||
do recs=1 while lines(ifid)\==0 /*keep reading records until finished. */
|
||||
rec=linein(ifid) /*read the next record (line) of file. */
|
||||
parse var rec datestamp Idata /*pick off the dateStamp and the data. */
|
||||
sum=0
|
||||
flg=0
|
||||
OKs=0
|
||||
|
||||
do j=1 until Idata='' /*process the instrument data. */
|
||||
do j=1 until Idata='' /*process the instrument data. */
|
||||
parse var Idata data.j flag.j Idata
|
||||
|
||||
if flag.j>0 then do /*if good data, ... */
|
||||
if flag.j>0 then do /*process good data ··· */
|
||||
OKs=OKs+1
|
||||
sum=sum+data.j
|
||||
if contFlag>longFlag then do
|
||||
longdate=datestamp
|
||||
longFlag=contFlag
|
||||
end
|
||||
contFlag=0
|
||||
if Cflag>Lflag then do
|
||||
Ldate=datestamp
|
||||
Lflag=Cflag
|
||||
end
|
||||
Cflag=0
|
||||
end
|
||||
else do /*flagged data ... */
|
||||
else do /*process flagged data ··· */
|
||||
flg=flg+1
|
||||
contFlag=contFlag+1
|
||||
Cflag=Cflag+1
|
||||
end
|
||||
end /*j*/
|
||||
|
||||
if OKs\==0 then avg=format(sum/OKs,,3)
|
||||
else avg='[n/a]'
|
||||
if OKs\==0 then avg=format(sum/OKs,,3)
|
||||
else avg='[n/a]'
|
||||
grandOKs=grandOKs+OKs
|
||||
_=right(comma(avg),w)
|
||||
_=right(commas(avg),w)
|
||||
grandSum=grandSum+sum
|
||||
grandFlg=grandFlg+flg
|
||||
if flg==0 then call sy datestamp ' average='_
|
||||
else call sy datestamp ' average='_ ' flagged='right(flg,2)
|
||||
if flg==0 then call sy datestamp ' average='_
|
||||
else call sy datestamp ' average='_ ' flagged='right(flg,2)
|
||||
end /*recs*/
|
||||
|
||||
recs=recs-1 /*adjust for reading end-of-file.*/
|
||||
if grandOKs\==0 then Gavg=format(grandsum/grandOKs,,3)
|
||||
recs=recs-1 /*adjust for reading the end─of─file. */
|
||||
if grandOKs\==0 then Gavg=format(grandSum/grandOKs,,3)
|
||||
else Gavg='[n/a]'
|
||||
call sy
|
||||
call sy copies('═',60)
|
||||
call sy ' records read:' right(comma(recs),w)
|
||||
call sy ' grand sum:' right(comma(grandSum),w+4)
|
||||
call sy ' grand average:' right(comma(Gavg),w+4)
|
||||
call sy ' grand OK data:' right(comma(grandOKs),w)
|
||||
call sy ' grand flagged:' right(comma(grandFlg),w)
|
||||
if longFlag\==0 then
|
||||
call sy ' longest flagged:' right(comma(longFlag),w) " ending at " longdate
|
||||
call sy ' records read:' right(commas(recs), w)
|
||||
call sy ' grand sum:' right(commas(grandSum), w+4)
|
||||
call sy ' grand average:' right(commas(Gavg), w+4)
|
||||
call sy ' grand OK data:' right(commas(grandOKs), w)
|
||||
call sy ' grand flagged:' right(commas(grandFlg), w)
|
||||
if Lflag\==0 then call sy ' longest flagged:' right(commas(Lflag),w) " ending at " Ldate
|
||||
call sy copies('═',60)
|
||||
exit /*stick a fork in it, we're done.*/
|
||||
/*──────────────────────────────────SY subroutine───────────────────────*/
|
||||
sy: procedure; parse arg stuff; say stuff
|
||||
if 1==0 then call lineout ofid,stuff
|
||||
return
|
||||
/*──────────────────────────────────COMMA subroutine────────────────────*/
|
||||
comma: procedure; parse arg _,c,p,t;arg ,cu;c=word(c ",",1)
|
||||
if cu=='BLANK' then c=' ';o=word(p 3,1);p=abs(o);t=word(t 999999999,1)
|
||||
if \datatype(p,'W')|\datatype(t,'W')|p==0|arg()>4 then return _;n=_'.9'
|
||||
#=123456789;k=0;if o<0 then do;b=verify(_,' ');if b==0 then return _
|
||||
e=length(_)-verify(reverse(_),' ')+1;end;else do;b=verify(n,#,"M")
|
||||
e=verify(n,#'0',,verify(n,#"0.",'M'))-p-1;end
|
||||
do j=e to b by -p while k<t;_=insert(c,_,j);k=k+1;end;return _
|
||||
exit /*stick a fork in it, we're all done. */
|
||||
/*────────────────────────────────────────────────────────────────────────────*/
|
||||
commas: procedure; parse arg _; n=_'.9'; #=123456789; b=verify(n,#,"M")
|
||||
e=verify(n,#'0',,verify(n,#"0.",'M'))-4
|
||||
do j=e to b by -3; _=insert(',',_,j); end /*j*/; return _
|
||||
/*────────────────────────────────────────────────────────────────────────────*/
|
||||
sy: say arg(1); call lineout ofid,arg(1); return
|
||||
|
|
|
|||
32
Task/Text-processing-1/Ruby/text-processing-1-2.rb
Normal file
32
Task/Text-processing-1/Ruby/text-processing-1-2.rb
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
Reading = Struct.new(:date, :value, :flag)
|
||||
|
||||
DailyReading = Struct.new(:date, :readings) do
|
||||
def good() readings.select(&:flag) end
|
||||
def bad() readings.reject(&:flag) end
|
||||
def sum() good.map(&:value).inject(0.0) {|sum, val| sum + val } end
|
||||
def avg() good.size > 0 ? (sum / good.size) : 0 end
|
||||
def print_status
|
||||
puts "%11s: good: %2d bad: %2d total: %8.3f avg: %6.3f" % [date, good.count, bad.count, sum, avg]
|
||||
self
|
||||
end
|
||||
end
|
||||
|
||||
daily_readings = IO.foreach(ARGV.first).map do |line|
|
||||
(date, *parts) = line.chomp.split(/\s/)
|
||||
readings = parts.each_slice(2).map {|pair| Reading.new(date, pair.first.to_f, pair.last.to_i > 0)}
|
||||
DailyReading.new(date, readings).print_status
|
||||
end
|
||||
|
||||
all_readings = daily_readings.flat_map(&:readings)
|
||||
good_readings = all_readings.select(&:flag)
|
||||
all_streaks = all_readings.slice_when {|bef, aft| bef.flag != aft.flag }
|
||||
worst_streak = all_streaks.reject {|grp| grp.any?(&:flag)}.sort_by(&:size).last
|
||||
|
||||
total = good_readings.map(&:value).reduce(:+)
|
||||
num_readings = good_readings.count
|
||||
puts
|
||||
puts "Total: %.3f" % total
|
||||
puts "Readings: #{num_readings}"
|
||||
puts "Average %.3f" % total./(num_readings)
|
||||
puts
|
||||
puts "Max run of #{worst_streak.count} consecutive false readings from #{worst_streak.first.date} until #{worst_streak.last.date}"
|
||||
63
Task/Text-processing-1/VBScript/text-processing-1.vb
Normal file
63
Task/Text-processing-1/VBScript/text-processing-1.vb
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
Set objFSO = CreateObject("Scripting.FileSystemObject")
|
||||
Set objFile = objFSO.OpenTextFile(objFSO.GetParentFolderName(WScript.ScriptFullName) &_
|
||||
"\data.txt",1)
|
||||
|
||||
bad_readings_total = 0
|
||||
good_readings_total = 0
|
||||
data_gap = 0
|
||||
start_date = ""
|
||||
end_date = ""
|
||||
tmp_datax_gap = 0
|
||||
tmp_start_date = ""
|
||||
|
||||
Do Until objFile.AtEndOfStream
|
||||
bad_readings = 0
|
||||
good_readings = 0
|
||||
line_total = 0
|
||||
line = objFile.ReadLine
|
||||
token = Split(line,vbTab)
|
||||
n = 1
|
||||
Do While n <= UBound(token)
|
||||
If n + 1 <= UBound(token) Then
|
||||
If CInt(token(n+1)) < 1 Then
|
||||
bad_readings = bad_readings + 1
|
||||
bad_readings_total = bad_readings_total + 1
|
||||
'Account for bad readings.
|
||||
If tmp_start_date = "" Then
|
||||
tmp_start_date = token(0)
|
||||
End If
|
||||
tmp_data_gap = tmp_data_gap + 1
|
||||
Else
|
||||
good_readings = good_readings + 1
|
||||
line_total = line_total + CInt(token(n))
|
||||
good_readings_total = good_readings_total + 1
|
||||
'Sum up the bad readings.
|
||||
If (tmp_start_date <> "") And (tmp_data_gap > data_gap) Then
|
||||
start_date = tmp_start_date
|
||||
end_date = token(0)
|
||||
data_gap = tmp_data_gap
|
||||
tmp_start_date = ""
|
||||
tmp_data_gap = 0
|
||||
Else
|
||||
tmp_start_date = ""
|
||||
tmp_data_gap = 0
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
n = n + 2
|
||||
Loop
|
||||
line_avg = line_total/good_readings
|
||||
WScript.StdOut.Write "Date: " & token(0) & vbTab &_
|
||||
"Bad Reads: " & bad_readings & vbTab &_
|
||||
"Good Reads: " & good_readings & vbTab &_
|
||||
"Line Total: " & FormatNumber(line_total,3) & vbTab &_
|
||||
"Line Avg: " & FormatNumber(line_avg,3)
|
||||
WScript.StdOut.WriteLine
|
||||
Loop
|
||||
WScript.StdOut.WriteLine
|
||||
WScript.StdOut.Write "Maximum run of " & data_gap &_
|
||||
" consecutive bad readings from " & start_date & " to " &_
|
||||
end_date & "."
|
||||
WScript.StdOut.WriteLine
|
||||
objFile.Close
|
||||
Set objFSO = Nothing
|
||||
Loading…
Add table
Add a link
Reference in a new issue