Crawler script

barreleye_ip=<ip>:8086 #cluster01
#barreleye_ip=<ip>:8086 #cluster02
output_file=barreleye_cluster01.ost.tables.new.csv

measurements=`curl -G "http://${barreleye_ip}/query?pretty=true" \
                --data-urlencode "db=barreleye" --data-urlencode \
                "q=SHOW measurements" | \
                grep '^                            \"' | sed -e 's,",,g'`
      
#measurements=`echo $measurements | sed -e "s, ,\n,g" | grep -e "md[t_]"`   
measurements=`echo $measurements | sed -e "s, ,\n,g" | grep -e "ost_"`

echo $measurements | sed -e "s, ,\n,g"

rm -f ${output_file}

#==============================================================
# To CSV
for m in $measurements
do
    # Get Tag keys -----------------------------------------
    tag_keys=`curl -G "http://${barreleye_ip}/query?pretty=true" \
              --data-urlencode "db=barreleye" --data-urlencode  \
              "q=SHOW TAG KEYS from \"$m\""`

    field_keys=`curl -G "http://${barreleye_ip}/query?pretty=true" \
                --data-urlencode "db=barreleye" --data-urlencode  \
                "q=SHOW FIELD KEYS from \"$m\""`

    tk=`echo $tag_keys | jq '.results[].series[].values[] | join(",")'`
    tk=`echo $tk | sed -e "s/ /,/g"`
    
    tag_values=`curl -G "http://${barreleye_ip}/query?pretty=true" \
                --data-urlencode "db=barreleye" --data-urlencode  \
                "q=SHOW TAG VALUES from \"$m\" WITH KEY in ($tk)"`

    
    echo `echo $tag_keys | jq '.results[].series[].name'`";" \
	 `echo $tag_keys | jq '.results[].series[].values | length'`";" \
	 `echo $tag_keys | jq '.results[].series[].values[]'`";" \
	 `echo $field_keys | jq '.results[].series[].values | length'`";" \
	 `echo $field_keys | jq '.results[].series[].values[]'`";" \
	 `echo $tag_values | jq '.results[].series[].values | length'`";" \
	 `echo $tag_values | jq '.results[].series[].values[]'` >>  ${output_file}

done

# Cleaned with regex tag values
# \[ "exp_client", "[0-9\.]*" \]   
# \[ "user_id", "[^"]*" \]
# \[ "ll_job_[ug]*id", "[^"]*" \]
# \[ "job_id", "[^"]*" \]

#==============================================================
# To WikiTable

output_file=barreleye_cluster01.ost.wiki.tables.new

rm -f ${output_file}
rm "crawl.log"

for m in $measurements
do
    echo '{| class="wikitable mw-collapsible"' >>   ${output_file}
    echo '|+Measurement: '"$m"                 >>   ${output_file}
    echo '!Key'                                >>   ${output_file}
    echo '!Value'                              >>   ${output_file}
    echo '!Explanation'                        >>   ${output_file}
    echo '|-'                                  >>   ${output_file}
    
    # Get Tag keys -----------------------------------------
    tag_keys=`curl -G "http://${barreleye_ip}/query?pretty=true" \
              --data-urlencode "db=barreleye" --data-urlencode  \
              "q=SHOW TAG KEYS from \"$m\""`

    tk=`echo $tag_keys | jq '.results[].series[].values[] | join(",")'`
    tk=`echo $tk | sed -e "s/ /,/g"`
   
    for i in `echo $tk | sed -e 's/,/ /g'`
    do
	tag_values=`curl -G "http://${barreleye_ip}/query?pretty=true" \
                         --data-urlencode "db=barreleye" --data-urlencode  \
                         "q=SHOW TAG VALUES from \"$m\" WITH KEY in ($i)"`

	num_tv=`echo $tag_values | jq '.results[].series[].values | length'`

	if  (( $num_tv < 2 )); then 
	    echo "|"`echo $i | sed -e 's/"//g'` >>  ${output_file}
	elif ((  $num_tv > 50 )); then
	    echo "$m, $i, $num_tv shortened to 10" >> "crawl.log"
	    num_tv=10
            echo "| rowspan=\"$num_tv\"|"`echo $i | sed -e 's/"//g'` >>  ${output_file}
	else
	    echo "| rowspan=\"$num_tv\"|"`echo $i | sed -e 's/"//g'` >>  ${output_file}
	fi
    
	for val in `echo $tag_values | jq '.results[].series[].values[range('$num_tv')][1]'` 
	do
	    echo "|"`echo $val | sed -e 's/"//g'` >>  ${output_file}
	    echo "|"                              >>  ${output_file}
	    echo "|-"                             >>  ${output_file}
	done
	    
    done
    sed -i -e '$ s/|-/|}/' $output_file
done