dispatch-macro-characters
(set-macro-character #\] (get-macro-character #\]) nil)
(defun separated-to-simple-integer (list)
(loop
as num in list
as val = num then (+ num (* val 1000))
finally (return val)))
(defun separated-integer-reader (stream subchar arg)
(declare (ignore subchar arg))
(funcall #'separated-to-simple-integer
(read-delimited-list #\] stream t)))
(set-dispatch-macro-character #\# #\[ #'separated-integer-reader)
CL-USER> #[12 345 678] 12345678 CL-USER> (* #[12 345 678] 2) 24691356 CL-USER>
multiconnection download with scsh
There’s a server (of a radio) I download audio files from. The thing is, that the bandwidth for a connection is limited to ~24Kb/sec nowdays (several years ago there wasn’t any limit). By getting the file with multiple connections and concurrently solves the problem somewhat. Unfortunately the number of connections from a given IP address is also limited to ~15. Anyway, let’s say ~240Kb/sec (when using 10 connections) is much more than 24Kb/sec.
Parts of a file can be obtained by Curl. I decided to use The Scheme Shell to implement my idea due to its thread support and strong relationship with command line tools (as being a shell).
The solution is a fast hack. Let’s see..
$ ls
getItFast.scm getItFast.scm~
$ ./getItFast.scm http://someserver/2200.mp3
$ ls
2200.mp3.00 2200.mp3.08 2200.mp3.16 2200.mp3.24 2200.mp3.32 2200.mp3.40
2200.mp3.01 2200.mp3.09 2200.mp3.17 2200.mp3.25 2200.mp3.33 2200.mp3.41
2200.mp3.02 2200.mp3.10 2200.mp3.18 2200.mp3.26 2200.mp3.34 2200.mp3.42
2200.mp3.03 2200.mp3.11 2200.mp3.19 2200.mp3.27 2200.mp3.35 getItFast.scm
2200.mp3.04 2200.mp3.12 2200.mp3.20 2200.mp3.28 2200.mp3.36 getItFast.scm~
2200.mp3.05 2200.mp3.13 2200.mp3.21 2200.mp3.29 2200.mp3.37
2200.mp3.06 2200.mp3.14 2200.mp3.22 2200.mp3.30 2200.mp3.38
2200.mp3.07 2200.mp3.15 2200.mp3.23 2200.mp3.31 2200.mp3.39
$ cat 2200.mp3.* > 2200.mp3
$ rm 2200.mp3.*
$ ls
2200.mp3 getItFast.scm getItFast.scm~
$ cat getItFast.scm
#!/usr/bin/scsh \
-o placeholders -o threads -o locks -s
!#
; this many thread will be started,
; each of'em represents a connection
(define POOL-SIZE 10)
; the length of a chunk in bytes
; (downloaded with one connection)
(define STEP 1000000)
(define URL (argv 1))
(define FNAME (file-name-nondirectory URL))
(define url-content-length
(lambda (url)
(string->number
(cadr ((infix-splitter (rx (+ white)))
(run/string
(| (curl -s -S -I ,url)
(grep "Content-Length"))))))))
(define LENGTH (url-content-length URL))
(define make-queue
(lambda (data-list)
(let ((lock (make-lock)))
(lambda ()
(let ((re '()))
(obtain-lock lock)
(if (null? data-list)
(set! re '())
(begin
(set! re (car data-list))
(set! data-list (cdr data-list))))
(release-lock lock)
re)))))
(define range-string
(lambda (beg end)
(let ((begs (number->string beg))
(ends (number->string end)))
(string-append begs "-" ends))))
(define get-part
(lambda (beg end fn)
(run (curl -o ,fn -s -S -r
,(range-string beg end) ,URL))))
; this long is the number field
; in the filenames of parts
(define PADLEN
(string-length
(number->string
(ceiling
(/ LENGTH STEP)))))
(define file-counter-string
(lambda (i)
(let loop ((s (number->string i)))
(if (<= PADLEN (string-length s))
s
(loop (string-append "0" s))))))
(define counted-file-name
(lambda (i)
(string-append FNAME
"."
(file-counter-string i))))
; this contains the works to do
; (work ~ download a specific chunk)
; e.g. ((0 999999 "foo.mp3.00") (1000000 1999999 "foo.mp3.01") ... )
(define QUEUE
(make-queue
(let loop ((work-list '()) (low 0) (upp (- STEP 1)) (counter 0))
(if (> low LENGTH)
work-list
(loop (cons (list low upp (counted-file-name counter)) work-list)
(+ upp 1)
(min LENGTH (+ upp STEP))
(+ counter 1))))))
(define signal-thread-finish
(lambda (waiter)
(placeholder-set! waiter #f)))
(define start-worker
(lambda ()
(let ((waiter (make-placeholder)))
(spawn
(lambda ()
(let loop ()
(let ((work (QUEUE)))
(if (null? work)
(signal-thread-finish waiter)
(begin
(apply get-part work)
(loop)))))))
waiter)))
(let loop ((i POOL-SIZE) (waiters '()))
(if (= i 0)
(map placeholder-value waiters)
(loop (- i 1) (cons (start-worker) waiters))))
$
Useful links:
hacking .NET assemblies
I’ve used grep and ed unix tools which are installed here as part of a cygwin, but you don’t need these tools. Grep is used only for looking into a source file and ed is used to edit that file which also can be done with your favourite editor.
The story is about changing the behaviour of a .NET executable file with .NET SDK tools but without access to the source by knowing how intermediate language (IL) instructions look like. Find relevant part here, Partition III, p. 73. (p. 397. in PDF).
In case of class library dlls the process works only if the user of these libraries don’t refer them by strong name. Strong named assemblies has a hash and/or a signature by which you cannot change the binary dll in such a way that the hash/signature remains the same.
C:\test>dir/b
sample.cs
C:\test>type sample.cs
public class Bar
{
public static void Main (string[] baz)
{
System.Console.WriteLine (1);
}
}
C:\test>csc sample.cs
Microsoft (R) Visual C# 2005 Compiler version 8.00.50727.3053
for Microsoft (R) Windows (R) 2005 Framework version 2.0.50727
Copyright (C) Microsoft Corporation 2001-2005. All rights reserved.
C:\test>del sample.cs
C:\test>dir/b
sample.exe
C:\test>sample.exe
1
C:\test>ildasm/out:sample.il sample.exe
C:\test>dir/b
sample.exe
sample.il
sample.res
C:\test>grep -A8 "static.*void.*Main" sample.il
.method public hidebysig static void Main(string[] baz) cil managed
{
.entrypoint
// Code size 9 (0x9)
.maxstack 8
IL_0000: nop
IL_0001: ldc.i4.1
IL_0002: call void [mscorlib]System.Console::WriteLine(int32)
IL_0007: nop
C:\test>ed sample.il
2075
/static.*void.*Main/
.method public hidebysig static void Main(string[] baz) cil managed
/WriteLine/
IL_0002: call void [mscorlib]System.Console::WriteLine(int32)
?ldc.i4?
IL_0001: ldc.i4.1
s/i4.1/i4.0/
w
2075
.
IL_0001: ldc.i4.0
q
C:\test>del sample.exe
C:\test>dir/b
sample.il
sample.res
C:\test>ilasm/quiet sample.il
C:\test>dir/b
sample.exe
sample.il
sample.res
C:\test>del sample.il
C:\test>del sample.res
C:\test>dir/b
sample.exe
C:\test>sample.exe
0
C:\test>
split mp3 with mencoder
Update: this solution uses an .avi file as auxiliary medium, but I’d like to emphasize that your mp3 won’t be reencoded in any sense. In the other hand, the id3 information won’t survive AFAIK. All in all, this is a hack. Normally you don’t use mencoder and mplayer to split mp3 files but it’s clearly possible. If you’re looking for a command line tool with which you can split your mp3 files without reencoding, feel free to use mp3splt which is also free. (Homepage is here.)
Let’s say your mp3 is 1804 seconds long. You need some jpeg file (some.jpg from now on) to use as video content because mencoder works only with video files regarding the split functionality.
First, create the list of frames. The only thing you need here is a file which contains the filename of your jpeg in as many lines as seconds long your mp3 is. On Linux machines or in a Cygwin shell you can do it like this:
$ seq 1 1804 | sed 's/^.*$/some.jpg/' > listfile.txt
Or in a cmd.exe window (probably on Windowns):
c:\splitdir>del listfile.txt c:\splitdir>for /l %i in (1,1,1804) do echo some.jpg >> listfile.txt
Let’s say your mp3 file is music.mp3:
mencoder "mf://@listfile.txt" -mf fps=1 \ -audiofile music.mp3 -oac copy -ovc lavc -o music.avi
Now you have a video which constantly shows your jpeg file and plays the mp3 file in the meanwhile. Mencoder can split this file now:
mencoder -ss 300 -endpos 5 \ -oac copy -ovc copy -o part.avi music.avi
You have the right (let’s say you need the 5:00 – 5:05 one) part, but as an avi file. With mplayer, you can dump the audio part:
mplayer -dumpaudio -dumpfile part.mp3 part.avi
Enjoy.U
cl-perec tutorial, lisp clos persistency
cl-perec is an RDBMS based CLOS persistency library. Consider these about a fresh install of a postgresql server before starting. Socket connections are refused by default. Then create a database for storing the instances:
$ sudo -u postgres psql
[sudo] password for grault:
Welcome to psql 8.3.7, the PostgreSQL interactive terminal.
Type: \copyright for distribution terms
\h for help with SQL commands
\? for help with psql commands
\g or terminate with semicolon to execute query
\q to quit
postgres=# create database perec_db1;
CREATE DATABASE
postgres=# create user perec_user with password 'perec999pass';
CREATE ROLE
postgres=# grant all on database perec_db1 to perec_user;
GRANT
postgres=# \q
$
and then load the following forms:
(asdf:oos 'asdf:load-op :cl-perec)
(cl-def:def defclass-star:class* myconn
(cl-perec:database-mixin cl-rdbms:postgresql-postmodern) ())
(setf cl-perec:*database*
(make-instance 'myconn
:generated-transaction-class-name 'transaction
:default-result-type 'vector
:muffle-warnings t
:connection-specification '(:host "localhost"
:port 5433
:database "perec_db1"
:user-name "perec_user"
:password "perec999pass")))
(cl-rdbms:start-sql-recording)
(cl-perec:defpclass c1 ()
((s1 :type string :initform "foo" :initarg :s1 :accessor s1-of)))
Finally issue some make-instance:
CL-USER> (cl-perec:with-transaction
(make-instance 'c1 :s1 "hello"))
; BEGIN
; SELECT relname FROM pg_class WHERE relkind = 'r'
; CREATE TABLE _c1 (_oid BIGINT NOT NULL PRIMARY KEY, _s1 TEXT)
; DROP VIEW IF EXISTS _c1_di
; CREATE VIEW _c1_di AS SELECT _oid FROM _c1
; DROP VIEW IF EXISTS _c1_dp
; CREATE VIEW _c1_dp AS SELECT _oid, _c1._s1 FROM _c1
; DROP VIEW IF EXISTS _c1_dd
; CREATE VIEW _c1_dd AS SELECT _oid, _c1._s1 FROM _c1
; DROP VIEW IF EXISTS _c1_ai
; CREATE VIEW _c1_ai AS SELECT _oid FROM _c1
; DROP VIEW IF EXISTS _c1_ap
; CREATE VIEW _c1_ap AS SELECT _oid, _c1._s1 FROM _c1
; DROP VIEW IF EXISTS _c1_ad
; CREATE VIEW _c1_ad AS SELECT _oid, _c1._s1 FROM _c1
; COMMIT
; BEGIN
; SELECT relname FROM pg_class WHERE relkind = 'S'
; CREATE SEQUENCE _instance_id
; SELECT NEXTVAL('_instance_id')
; $1 = 121174 as BIGINT, $2 = hello as TEXT
; INSERT INTO _c1 (_oid, _s1) VALUES ($1::BIGINT, $2::TEXT)
; COMMIT
#<C1 :persistent #t 1>
CL-USER> (cl-perec:with-transaction
(make-instance 'c1 :s1 "hello2"))
; BEGIN
; SELECT NEXTVAL('_instance_id')
; $1 = 186710 as BIGINT, $2 = hello2 as TEXT
; INSERT INTO _c1 (_oid, _s1) VALUES ($1::BIGINT, $2::TEXT)
; COMMIT
#<C1 :persistent #t 2>
CL-USER>
blinking flower with ImageMagick
There’s an interesting effect in this video, from 4:20 – 4:45. Here’s a script which generates effects like that.
$ cat flower.sh
#!/bin/bash
W=60
H=20
IW=400
IH=400
DELAY=8
ARCDELTA=20
DIST=60
MUL=$[$DIST + $[$W / 2]]
convert -size ${W}x${H} xc:black \
-fill black -stroke red \
-draw "polygon 1,1,$[$W - 1],$[$H / 2],1,$[$H - 2]" tria_empty.png
convert -size ${W}x${H} xc:black \
-fill red -stroke red \
-draw "polygon 1,1,$[$W - 1],$[$H / 2],1,$[$H - 2]" tria_full.png
convert -size ${IW}x${IH} xc:black background.png
cp background.png bg0.png
cp background.png bg1.png
cp background.png bg2.png
seq 0 $ARCDELTA 360 | \
sed "1d" | \
sed "s/^\(.*\)$/${MUL}*c(\1\*a(1)\/45)\n${MUL}*s(\1*a(1)\/45)\n\1/" | \
bc -l | \
sed 's/$/\/1/' | \
bc | \
sed -n -e 'N;N;s/\n/;/g;p' | \
sed 's/^/composite -gravity center -compose Plus -geometry +/' | \
sed 's/;/+/' | sed 's/+-/-/g' | \
sed 's/;/ tria_full.png -gravity center -rotate /' | \
sed 's/$/ bg2.png bg2.png/' | \
sed -n -e 'p;p' | \
sed -n -e 's/bg2/bg1/g;s/_full/_empty/;N;p' | \
sh
convert -delay $DELAY -loop 0 bg*.png animgif.gif
$
the result
word docx file from lisp
Due to the rdnzl .net layer for common lisp, you’re able to use the word interop assemblies and do full word or office automation through lisp. There’s only an Excel query type example on that page. This is a creational pattern for Word:
C:\notes\my-asdf-packs\cl-docx-sandbox>type cl-docx-sandbox.asd
(defsystem cl-docx-sandbox
:components ((:file "just-do-it"))
:depends-on (rdnzl cl-def))
C:\notes\my-asdf-packs\cl-docx-sandbox>type just-do-it.lisp
(defpackage :cl-docx-sandbox
(:use :cl :cl-def)
(:export #:one-liner-word))
(in-package :cl-docx-sandbox)
(def load-time-constant +miss+ (rdnzl:field "System.Reflection.Missing" "Value"))
(def function word-save-as (doc filename)
(rdnzl:invoke doc "SaveAs"
(rdnzl:ref filename)
(rdnzl:ref +miss+) (rdnzl:ref +miss+) (rdnzl:ref +miss+)
(rdnzl:ref +miss+) (rdnzl:ref +miss+) (rdnzl:ref +miss+)
(rdnzl:ref +miss+) (rdnzl:ref +miss+) (rdnzl:ref +miss+)
(rdnzl:ref +miss+) (rdnzl:ref +miss+) (rdnzl:ref +miss+)
(rdnzl:ref +miss+) (rdnzl:ref +miss+) (rdnzl:ref +miss+)))
(def (function e) one-liner-word (filename line)
(progn
(rdnzl:import-types "Microsoft.Office.Interop.Word"
"Document" "ApplicationClass" "Documents" "DocumentClass")
(let* ((app (rdnzl:new "Microsoft.Office.Interop.Word.ApplicationClass"))
(doc (rdnzl:invoke (rdnzl:property app "Documents") "Add"
(rdnzl:ref +miss+) (rdnzl:ref +miss+)
(rdnzl:ref +miss+) (rdnzl:ref +miss+)))
(sel (rdnzl:property app "Selection")))
(progn
(rdnzl:invoke sel "TypeText" line)
(rdnzl:invoke sel "TypeParagraph")
(word-save-as doc filename)
(rdnzl:invoke doc "Close"
(rdnzl:ref +miss+) (rdnzl:ref +miss+) (rdnzl:ref +miss+))))))
C:\notes\my-asdf-packs\cl-docx-sandbox>
Usage:
CL-USER> (asdf:oos 'asdf:load-op :cl-docx-sandbox) ; lots of stuff ; [...] CL-USER> (cl-docx-sandbox:one-liner-word "c:\\one-liner.docx" "Hello World!") :VOID CL-USER>
The result is:
one-liner.docx
join & filter multiline data records with sed
Grouping with sed:
$ cat nrs.txt 01 02 03 04 05 06 07 08 09 10 11 12 $ cat nrs.txt | sed -n -e 'N;s/\n/ /g;p' 01 02 03 04 05 06 07 08 09 10 11 12 $ cat nrs.txt | sed -n -e 'N;N;s/\n/ /g;p' 01 02 03 04 05 06 07 08 09 10 11 12 $ cat nrs.txt | sed -n -e 'N;N;N;s/\n/ /g;p' 01 02 03 04 05 06 07 08 09 10 11 12 $ cat nrs.txt | sed -n -e 'N;N;N;N;N;s/\n/ /g;p' 01 02 03 04 05 06 07 08 09 10 11 12 $
Good for the following task:
$ cat entries.txt entry-1-data-1 entry-1-data-2 entry-2-data-1 entry-2-data-2 $ cat entries.txt | sed -n -e 'N;s/\n/ /;p' entry-1-data-1 entry-1-data-2 entry-2-data-1 entry-2-data-2 $
Two long groups, removing elements (1st, 2nd):
$ cat nrs.txt | sed -n -e 'p;n' 01 03 05 07 09 11 $ cat nrs.txt | sed -n -e 'n;p' 02 04 06 08 10 12 $
In general case you have groups of length _k_ and the starting pattern is ‘n;n;…;n’. There’s k-1 number of letter n here. You can place letter p around the n-s, which is exactly k possibility. If there’s a p in position k0 then the k0-th element will be printed out.
So if you have 6 long groups and you want every fifth element:
$ cat nrs.txt | sed -n -e 'n;n;n;n;p;n' 05 11 $
office open xml file sample
Here’s the spec.
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<?mso-application progid="Word.Document"?>
<pkg:package xmlns:pkg="http://schemas.microsoft.com/office/2006/xmlPackage">
<pkg:part
pkg:name="/_rels/.rels"
pkg:contentType="application/vnd.openxmlformats-package.relationships+xml">
<pkg:xmlData>
<Relationships
xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship
Id="foo"
Type=
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"
Target="/word/document.xml"/>
</Relationships>
</pkg:xmlData>
</pkg:part>
<pkg:part
pkg:name="/word/document.xml"
pkg:contentType=
"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml">
<pkg:xmlData>
<w:document
xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
<w:body>
<w:p>
<w:r>
<w:t>Hello World</w:t>
<w:br />
<w:yearLong />
</w:r>
</w:p>
</w:body>
</w:document>
</pkg:xmlData>
</pkg:part>
</pkg:package>
cumulating minutes begun
$ cat seconds.txt 120 123 $ cat seconds.txt | sed 's/$/ 60 ~ 0 !=r +/' | sed '1i[1+] sr 0' | sed '$ap' | dc 5 $
