I. Shell Environment

1. show debug info

set -x      # var replaced, has a +
set -v      # var not replaced, no +

2. profile call stack

/etc/profile
.bash_profile
    .bashrc
        /etc/bashrc

3. list variables

set         # for current Shell and User
env         # for current User
export      # for current env, and sub processes inheritance

4. special variables

$$          # Process ID of the current process
$?          # Exit Status of the recent foreground process
$!          # Process ID of the last background job started
$#          # Number of prarameters

5. directory operation

~           # home dir
~+          # current working dir
-           # previous working dir

6. command substitution

echo $(command)
echo `command`

7. arithmetic expansion

echo $(( 3 + 2 ))
expr 3 + 2

II. GREP - Global Regular Expression Print

1. grep options

grep [OPTIONS] PATTERN [FILE...]
grep -i     # ignore case
grep -l     # print Filename instead of matched lines
grep -n     # print line number also
grep -w     # filter word

2. pattern matching

^           # line begin
$           # line end
\<          # word begin
\>          # word end
-w          # word

III. SED - Stream Editor

1. commands

sed [options] '[addresses] action [args]' files [ > outfile]
d           # deletes a line or lines
p           # prints a line or lines
r           # reads a file
s           # substitutes one string for another
w           # writes to a file

2. options

-f          # reads sed commands from a script file
-i          # in-place processing
-n          # Suppresses the default output

3. printing

vi data.file
northwest  NW Joel Craig       3.0 .98      3      4
western    WE Sharon Kelly     5.3 .97      5      23
southwest  SW Chris Foster     2.7 .8       2      18
southern   SO May Chin         5.1 .95      4      15
southeast  SE Derek Johnson    5.0 .70      4      17
eastern    EA Susan Beal       4.4 .8       5      20
northeast  NE TJ Nichols       5.1 .94      3      13
north      NO Val Shultz       4.5 .89      5      9
central    CT Sheri Watson     5.7 .94      5      13

# <bgn> and <,end> can use a number or a regular expression, or nothing
sed -n '<bgn><,end>p'       data.file
sed -n 'p'                  data.file
sed -n '3p'                 data.file
sed -n '3,5p'               data.file
sed -n '/north/p'           data.file
sed -n '/north/,/south/p'   data.file

4. substitution

# <g> is optional, for global
# "&" can used in <new> to refer to <old>
sed 's/<old>/<new>/<g>'     data.file

5. delete lines

sed '<bgn><,end>d'          data.file
sed '4,8d'                  data.file
sed '/west/d'               data.file

6. reading from replacement file

sed '/<regex>/r <r_file>'   data.file
sed '/north/r northmsg'     data.file

7. read from command file

vi command.sed
1,4d
s/north/North/
s/^east/East/

sed -f command.sed          data.file

8. write to output files

vi command.sed
/north/w northregions                   # write lines containing "north", to file northregions
s/9[0-9]/& Great job!/w topperformers   # write lines containing 9x, to file topperformers, with "Great job!" inserted.

sed -n -f command.sed       data.file

IV. AWK - Aho Weinberger and Kernighan

1. Command format

awk -f command.awk          data.file
awk 'statement'             data.file

# statement shape
pattern { ACTION }          # the action is taken on those records that match the pattern.
pattern                     # all records that match the pattern are printed.
{ ACTION }                  # the action is taken on all records in the input file.

2. Use Reg Expression

awk '/east/ { print $1, $5, $4 }' data.file
awk '/\.9/'                 data.file

3. Special patterns

BEGIN       # An action to take BEFORE reading any lines
END         # An action to take AFTER all lines are read processed

awk 'BEGIN { print "Eastern Regions Report Begin\n" }
/east/ { print "\t" $5, $4 }
END { print "\nEastern Region Report End" }' data.file

4. Use awk script file

vi report.awk
BEGIN { print "Eastern Regions\n" }
/east/ { print "\t" $5, $4 }
END {print "\nEastern Region Monthly Report"}

awk -f report.awk data.file

5. Built-In Variables

FS          # space or tab The input field separator Space or tab The input field separator
OFS         # space The output field separator
NR          # the number of records from the beginning of the first input file
NF          # the number of fields
length()    # function for length of field(s)
printf      # print formatted string


awk -F: '{ print $1, $3 }' /etc/group
awk 'BEGIN { FS=":" }; { print $1, $3 }' /etc/group
awk 'BEGIN { OFS="\t" }; {print $3,$4,$2}' data.file
awk 'BEGIN { OFS="\t" }; {print NR,$3,$4,$2}' data.file
awk '{ print length($0), $0 }' /etc/passwd
awk '{ printf "%-15s %3d \n", $3 " " $4, $7 }' data.file


vi report.awk
{ fn = $3 }
{ ln = $4 }
{ area = $2 }
{ printf "Name: %-15s Region: %2s\n", fn " " ln, area}

awk -f report.awk data.file

V. Interactive Scripts

1. printf

printf  FORMAT ARGUMENTS…

2. read

  • variables assignment

    • If there are more tokens than variables, the last variable holds all remaining tokens.
    • If there are more variables than tokens, the extra variables are assigned a null value.
    • If no variable names are supplied to the read command, the bash shell uses the values supplied in the REPLY variable.
  • capture command result

var=`ls -l /etc/passwd`

ls -l /etc/passwd > file
read v1 v2 v3 v4 v5 v6 v7 v8 < file


#!/bin/bash
# Script name: io2.sh
# This script prompts for input and prints messages
# involving the input received.
printf "Enter your name: "
read name junk
printf "Hi $name, how old are you? "
read age junk
printf "\n\t$age is an awkward age, $name,"
printf " You’re too old to depend on your parents,"
printf "and not old enough to depend on your children.\n\n"

3. file descriptor

  • 0 (stdin)
  • 1 (stdout)
  • 2 (stderr)
  • 3 ~ 9 are for programmer-defined file descriptors
exec N> filename
exec N< filename

command >&N
command <&N


#!/bin/bash
cp /etc/hosts /tmp/hosts2
grep -v '^#' /tmp/hosts2 > /tmp/hosts3

exec 3< /tmp/hosts3         # define fd 3 for input file /tmp/hosts3
exec 4> /tmp/hostsfinal     # define fd 4 for output file /tmp/hostsfinal

# read from fd 3 line by line
read <&3 addr1 name1 alias1
read <&3 addr2 name2 alias2
read <&3 addr3 name3 alias3

# alias would flexiablly include multi words, need be closed in quotations
printf "%-20s %-20s %-50s\n" $name1 $addr1 "$alias1"
printf "%-20s %-20s %-50s\n" $name2 $addr2 "$alias2"
printf "%-20s %-20s %-50s\n" $name3 $addr3 "$alias3"

exec 3<&-                   # close fd 3
exec 4<&-                   # close fd 4

4. here document

command << Keyword
input1
input2
...
Keyword


#!/bin/bash
printf "%s\n" "Select a terminal type"
cat << ENDINPUT
sun
ansi
wyse50
ENDINPUT
printf "Which would you prefer? "
read termchoice
printf "%s\n" "You choice is terminal type: $termchoice"

VI. Variables and Position Parameters

1. Variables

# removing portions of a string
${str_var%pattern}  # removes the smallest right-most substring of string str_var that matches pattern
${str_var%%pattern} # removes the largest right-most substring of string str_var that matches pattern
${str_var#pattern}  # removes the smallest left-most substring of string str_var that matches pattern
${str_var##pattern} # removes the largest left-most substring of string str_var that matches pattern

# define an integer
typeset -i num
num=10
printf "%d\n" $num

# define a readonly variable
readonly str="HELLO"
str="HelloHello"    # error: str: readonly variable
printf "%s\n" $str


# define an array
declare -a integer arr
arr[0]=10
arr[1]=21
arr[2]=755

printf "%d\n" ${#arr[*]}    # num of arr elements
printf "%d "  ${arr[*]}     # all elements
printf "\n"

2. Position Parameters

$0          # the name of the script
$1          # the first argument to the script
$2          # the second argument to the script
$#          # the number of arguments to the script
$@          # a list of all arguments to the script where each parameter is seen as a word
$*          # a list of all arguments to the script where each parameter is quoted as a string

#!/bin/bash
# ./argtest.sh a b c d e f g h i j k l m n
echo '$#: ' $#                              # $#:  14
echo '$@: ' $@                              # $@:  a b c d e f g h i j k l m n
echo '$*: ' $*                              # $*:  a b c d e f g h i j k l m n
echo '$1 $2 $9 $10 are: ' $1 $2 $9 ${10}    # $1 $2 $9 $10 are:  a b i j
shift
echo '$#: ' $#                              # $#:  13
echo '$@: ' $@                              # $@:  b c d e f g h i j k l m n
echo '$*: ' $*                              # $*:  b c d e f g h i j k l m n


#!/bin/bash
printf "Executing script %s\n" "$0"
printf "Executing script %s\n" "$*"
set uno duo tres                            # this overwrites CLI params
printf "One two three in Latin is: %s %s %s\n" "$1" "$2" "$3"
textline="name phone address birthdate salary"
set $textline                               # set some pos-params thru words of line
printf "%s\n" "$*"
printf "At this time \$1 = %s and \$4 = %s \n" "$1" "$4"
printf "%s %s\n" "$0" "$*"
set --                                      # unset those pos-params
printf "%s %s\n" "$0" "$*"                  # only left $0