Crawler script
Jump to navigation
Jump to search
barreleye_ip=<ip>:8086 #cluster01 #barreleye_ip=<ip>:8086 #cluster02 output_file=barreleye_cluster01.ost.tables.new.csv measurements=`curl -G "http://${barreleye_ip}/query?pretty=true" \ --data-urlencode "db=barreleye" --data-urlencode \ "q=SHOW measurements" | \ grep '^ \"' | sed -e 's,",,g'` #measurements=`echo $measurements | sed -e "s, ,\n,g" | grep -e "md[t_]"` measurements=`echo $measurements | sed -e "s, ,\n,g" | grep -e "ost_"` echo $measurements | sed -e "s, ,\n,g" rm -f ${output_file} #============================================================== # To CSV for m in $measurements do # Get Tag keys ----------------------------------------- tag_keys=`curl -G "http://${barreleye_ip}/query?pretty=true" \ --data-urlencode "db=barreleye" --data-urlencode \ "q=SHOW TAG KEYS from \"$m\""` field_keys=`curl -G "http://${barreleye_ip}/query?pretty=true" \ --data-urlencode "db=barreleye" --data-urlencode \ "q=SHOW FIELD KEYS from \"$m\""` tk=`echo $tag_keys | jq '.results[].series[].values[] | join(",")'` tk=`echo $tk | sed -e "s/ /,/g"` tag_values=`curl -G "http://${barreleye_ip}/query?pretty=true" \ --data-urlencode "db=barreleye" --data-urlencode \ "q=SHOW TAG VALUES from \"$m\" WITH KEY in ($tk)"` echo `echo $tag_keys | jq '.results[].series[].name'`";" \ `echo $tag_keys | jq '.results[].series[].values | length'`";" \ `echo $tag_keys | jq '.results[].series[].values[]'`";" \ `echo $field_keys | jq '.results[].series[].values | length'`";" \ `echo $field_keys | jq '.results[].series[].values[]'`";" \ `echo $tag_values | jq '.results[].series[].values | length'`";" \ `echo $tag_values | jq '.results[].series[].values[]'` >> ${output_file} done # Cleaned with regex tag values # \[ "exp_client", "[0-9\.]*" \] # \[ "user_id", "[^"]*" \] # \[ "ll_job_[ug]*id", "[^"]*" \] # \[ "job_id", "[^"]*" \] #============================================================== # To WikiTable output_file=barreleye_cluster01.ost.wiki.tables.new rm -f ${output_file} rm "crawl.log" for m in $measurements do echo '{| class="wikitable mw-collapsible"' >> ${output_file} echo '|+Measurement: '"$m" >> ${output_file} echo '!Key' >> ${output_file} echo '!Value' >> ${output_file} echo '!Explanation' >> ${output_file} echo '|-' >> ${output_file} # Get Tag keys ----------------------------------------- tag_keys=`curl -G "http://${barreleye_ip}/query?pretty=true" \ --data-urlencode "db=barreleye" --data-urlencode \ "q=SHOW TAG KEYS from \"$m\""` tk=`echo $tag_keys | jq '.results[].series[].values[] | join(",")'` tk=`echo $tk | sed -e "s/ /,/g"` for i in `echo $tk | sed -e 's/,/ /g'` do tag_values=`curl -G "http://${barreleye_ip}/query?pretty=true" \ --data-urlencode "db=barreleye" --data-urlencode \ "q=SHOW TAG VALUES from \"$m\" WITH KEY in ($i)"` num_tv=`echo $tag_values | jq '.results[].series[].values | length'` if (( $num_tv < 2 )); then echo "|"`echo $i | sed -e 's/"//g'` >> ${output_file} elif (( $num_tv > 50 )); then echo "$m, $i, $num_tv shortened to 10" >> "crawl.log" num_tv=10 echo "| rowspan=\"$num_tv\"|"`echo $i | sed -e 's/"//g'` >> ${output_file} else echo "| rowspan=\"$num_tv\"|"`echo $i | sed -e 's/"//g'` >> ${output_file} fi for val in `echo $tag_values | jq '.results[].series[].values[range('$num_tv')][1]'` do echo "|"`echo $val | sed -e 's/"//g'` >> ${output_file} echo "|" >> ${output_file} echo "|-" >> ${output_file} done done sed -i -e '$ s/|-/|}/' $output_file done