# Loop over and analyse the web logs
for i in `seq 11 -1 0`; do webalizer -o output -p -D /var/tmp/dnscache -N 4 -p -n www.pisoftware.com -r www.pisoftware.com ./www.pisoftware.com-access.log.$i ; done

# Remove all local addresses from web logs
for i in `seq 12 -1 0`; do grep -h -v ^203.20.51. www.pisoftware.com-access.log.$i >> access.log; done

# Print out all matching fields that occur more than once
perl -ane '$D{$F[0]}++; } { foreach (keys %D) { print "$_\n" if ($D{$_} > 1) } ;' foo

# Replace all ips with their hostnames, if possible
perl -MSocket -pe's/^([0-9.]+)/(gethostbyaddr(inet_aton($1),AF_INET))[0]||$1/e' file

# Find out how many instances and what hosts are connecting to a certain port
grep :1433 /var/log/kern.log | perl -ane '($host,$port) = $F[11] =~ m/^(.*):(.*)$/; print "$host\n";' | sort | uniq -c

# Replace all instances of a word with another
perl -pi.bak -e 's/index.htm\"/index.html\"/' *.html

# Remove a line that matches a regex
perl -lane 'print unless m/<regex>/' <file>

# Pull out all hits from SA in spam box
perl -lane '$foo = $_; if ($foo =~ /\((.*) hits/) { print "$1" };' Spam

# Add in another link on the menu bar on my website
perl -pi -e 's!<a href="http://lwn.net">LWN</a><br />!<a href=\"http://lwn.net/\">LWN</a><br />\n<span class=\"hide\">[<a href=\".\">i</a>]</span>\n<a href=\"http://www.kerneltrap.org/\">KernelTrap</a><br />\n!' *.html

# Summary from evaluation.log
perl -MSocket -F' :: ' -ane 'print "$F[2] from $F[3] <$F[0]> at ", (gethostbyaddr(inet_aton($F[5]),AF_INET))[0]||$F[5] ," downloaded $F[6] $F[7] $F[8] $F[9]\n"' ./evaluation.log

# Check what MUAs are being used
egrep 'User-Agent|X-Mailer' mbox | perl -F/:/ -ane 'chomp($F[1]); $tmp{$F[1]}++; } { foreach $mua (sort keys(%tmp)) { print "$mua: $tmp{$mua}\n"; }'

# Match [ABCD-abcd] at end of line
perl -lne 'if (/ne 'if (/\[([A-Z]{4}-[a-z]{4})\]$/) { print $1; }'

# Cat all log files to one
for i in `seq 23 -1 1`; do zcat /var/log/apache/staff.pisoftware.com-access.log.$i.gz >> staff.log; done
cat /var/log/apache/staff.pisoftware.com-access.log.0 >> staff.log
cat /var/log/apache/staff.pisoftware.com-access.log >> staff.log
for i in Oct Nov Dec Jan Feb Mar Apr May; do (grep "/$i/" staff.log > staff.$i.log); done
for i in Oct Nov Dec Jan Feb Mar Apr May; do webalizer -c /etc/webalizer/staff.pisoftware.com staff.$i.log; done

for i in www.qinvest.com.au www.queenslandsugar.com www.srdc.gov.au www.pisoftware.com.au; do
    for j in `seq 30 -1 1`; do
        zcat $i-access.log.$j.gz >> /var/tmp/logs/$i.log;
    done;
    cat $i-access.log.0 >> /var/tmp/logs/$i.log;
    cat $i-access.log >> /var/tmp/logs/$i.log;
done

# Move *.1600.jpg to *.jpg
for i in *; do cp $i ../`basename $i .1600.jpg`.jpg; done

# Convert from one format to another
for i in *png; do convert $i ${i%png}eps; done

[03:09PM] <evilRasp> Jiko: hdparm -u 1 -d 1 -A 1 -a -c 1 /dev/hde :)

Evil hack to see what files people have downloaded
zcat access.log.gz | grep username; done | perl -ane 'print "$F[4] $F[6]\n"' | sort -rn | less

awk '/default.ida/ {print $3}' /var/log/squid/access.log

egrep -h 'root.exe|cmd.exe| 408 ' *access* | perl -ane 'print "$F[0]\n"' | sort | uniq | wc -l
grep WEB-IIS /var/log/auth.log | perl -ne 'm/\]: ([0-9.:]+) -> (.*)$/; $in = $1; $us = $2; @foo = split(/:/, $in); print "$foo[0]\n";' | sort | uniq | wc -l

# Split into 1G sized files
dd if=/dev/hda1 | split --bytes=1000m - outputfile

# Backing up a raw device
# On laptop
dd if=/dev/hda1 | gzip -f - | nc carmack 5050
# On server
nc -l -p 5050 | gzip -d -f > filename

# find all lists that have a .config but not a subscribers file
for i in *.config; do if [ ! -e ${i%.config} ]; then echo ${i%.config}; fi; done

Solaris
-------
who -r tells what runlevel you're in on Solaris

Sol CPU stuff
mpstat 3
prtdiag -v
uname -X
prstat

pkgrm `pkginfo | grep IBM | cut -d" " -f 2`

for i in *.tar; do
   nam=`basename $i .tar`;
   tar xvf $i;
   cd $nam;
   patchadd .;
   cd ..;
   rm -r $nam;
done 

When installing a patch cluster, a return code of 2 or 8 is ok - means either already applied or not needed

pkill = equivalent of Linux's killall

/etc/nodename = hostname
/etc/defaultrouter = default route ip address
/etc/defaultdomain = domain name
/etc/inet/* = network config stuff
/etc/gateways = router
/etc/notrouter = multihomed host, not a router

From ok prompt, boot -s to boot single user

/usr/bin/showrev -p shows patch revisions

snoop = equivalent of tcpdump

Network stuff
-------------
netstat -s = tcp/ip stats
netstat -i = interface stats
netstat -r = routing table

SQL
---
Postgresql
----------
$ sudo -u postgres psql

select * from pg_user;
alter user username with password 'passwd';

Mysql
-----
# mysql -u root mysql
mysql>  UPDATE user SET Password=PASSWORD ('new_password') WHERE user='root';
mysql> FLUSH PRIVILEGES; 

# Let bugs@carmack.bne.pisoftware.com have all privileges on bugs db
GRANT ALL PRIVILEGES on bugs.* to bugs@carmack.bne.pisoftware.com;

# See what databases
mysql> select database();

# See whats inside the currently selected db
mysql> show tables;

# See what a table is
mysql> describe <tablename>;

SSL
---
$ openssl genrsa -des3 -rand /var/log/messages.4:/var/log/lastlog:/var/log/cron.3:/usr/local/apache/logs/ssl_engine_log.2.gz:/usr/local/apache/logs/access_log.4.gz -out /var/tmp/conveyancing.pisoftware.com.key 1024
$ openssl req -new -key /var/tmp/conveyancing.pisoftware.com.key -out /var/tmp/conveyancing.pisoftware.com.csr
$ openssl x509 -req -days 30 -in /var/tmp/conveyancing.pisoftware.com.csr  -signkey /var/tmp/conveyancing.pisoftware.com.key -out /var/tmp/conveyancing.pisoftware.com.crt

# Check how many configs have appropriate options set, regardless of if they're modules or not
grep -hv "^$" *2.4* | grep -v "^#" | sed 's/=.$//g' | sort | uniq -c | sort -rn | sort -b -k 2 | less

Real
----
# Encoding
$ ./realproducer -l 0,6 -s carmack:4040/foo.rm -y 0 -vl 5 -v 2 -vx 320,240 -u admin -p C0ffee! -t 5 

use http://host:8080/ramgen/encoder/foo.rm

Hardware
--------
Use scsi bios (usually ctrl-a) on boot to change IRQ
Don't forget to reserve irq in pci / pnp section of bios

# To get kernel version on all hosts
for i in lee cheswick clarke raymond stallman babbage minter cox adams abrash knuth brin carmack asimov pike joy skud kildall zaphod trillian zwicky romero hopper rusty miller conway morris; do ssh -x $i uname -a | perl -ane 'print "$F[1]: $F[2]\n"'; done

Spider
------
spider --file /spare/gallery.xml --filePath /spare/pics/ --inputImagePath /cdrom/pictures/20020709-coronation_drive/

Exim
----
for i in `grep -l -- -frozen /var/spool/exim/input/* | cut -d- -f0-3`; do mv $i* /var/tmp/foo; done

File extensions
---------------
# find out how many of each extension files there is
find . -type f | sed -e 's/^.*\.\([^.]*\)$/\1/g' | tr [A-Z] [a-z] | sort | uniq -c
# get the size for each extension
for i in `cat /var/tmp/list2.txt`; do echo -n $i; find . -type f -iname \*.$i -exec du -s {} \; | perl -ane '$tot += $F[0]; } { print ": $tot\n";'; done > /var/tmp/totals.txt
# find the first file of each extension
for i in `cat /var/tmp/list2.txt`; do find . -type f -iname \*.$i | head -1; done
# copy the first file of each extension to a peer directory
for i in `cat /var/tmp/list2.txt`; do cp -p --parents "`find . -type f -iname \*.$i | head -1`" /spare/doar/data-subset/; done

# Find out what deleted files are open
$ sudo lsof +L1

Backups
-------
# tar cvf - /Medic | ssh bmarshal@carmack "cat - > /spare/medic.tar"

GPG
---
$ for i in `gpg --list-keys | grep pub | perl -ane '$foo = (split(/\//, $F[1]))[1]; print "$foo\n"';`; do gpg --recv-key $i; done

$ for i in `for i in \`gpg --list-public-keys | grep pub | perl -ane '$foo = (split(/\//, $F[1]))[1]; print "$foo\n"';\`; do gpg --textmode --list-sigs $i; done | grep sig | perl -ane 'if (length($F[1]) == 1) { print "$F[2]\n"; } else { print "$F[1]\n" }';`; do gpg --recv-key $i; done

$ for i in `for i in \`gpg --list-public-keys | grep pub | perl -ane '$foo = (split(/\//, $F[1]))[1]; print "$foo\n"';\`; do gpg --textmode --list-sigs $i; done | grep sig | perl -ane 'if (length($F[1]) == 1) { print "$F[2]\n"; } else { print "$F[1]\n" }' | sort | uniq;`; do gpg --recv-key $i; done

Distcc
------
time MAKE="make -j3" DISTCC_HOSTS="carmack trillian brin" fakeroot make-kpkg binary-arch

Samba
-----
perl -F/:/ -ane  'print "/usr/local/sbin/smbldap-useradd -a -u $F[2] -g $F[3] -n $F[0]\n";' users-id.txt

VIM
---
Marking:
m<letter> to mark, '<letter> to reference it
ie, mm and mn at two points
:'m,'ns/blah/foo/

Folds:
#zF folds # lines
zo opens again
:help zf for more info
With marking, :'m,'nfo will fold between those two marks

Visual:
^V to go to visual mode, hilight what you want
Hit : then any commands are applied to the highlighted bit

Upper case first letter of each line:
%s!^.!\U\0!g
\L is lower case

Exim
----
Check how an address will be delivered
exim -bt <address>

Force delivery of a stuck message, using debugging
exim -d -M <message-id>

sendmail -Mt <message id> to thaw a message
sendmail -Mvl <mess id> to view the log
         -Mvh to view header
         -Mvb to view body
sendmail -Mrm to remove the queued message
sendmail -Mar <message id> <address> to add a recipient to a message