davideaves.com - Page 2 of 13 - Live in a world of your own, but always welcome visitors.davideaves.com | Live in a world of your own, but always welcome visitors.

Ansible playbook to provision Netscaler VIPs.

19. December 2018 · Comments Off · Categories: Ansible, Linux, Linux Admin, Load Balancing, NetScaler, Networking · Tags: ansible, netscaler, nitrosdk-python, playbook

The following playbook will create a fully functional VIP; including the supporting monitor, service-group (pool) and servers (nodes) on a netscaler loadbalancer. Additionally, the same playbook has the ability to fully deprovision a VIP and all its supporting artifacts. To do all this I use the native Netscaler Ansible modules. When it comes to using the netscaler_servicegroup module, since the number of servers are not always consistent; I create that task with a Jinja2 template, where its imported back into the play.

netscaler_provision.yaml: a /usr/bin/ansible-playbook -f 10 script text executable, ASCII text

#!/usr/bin/ansible-playbook -f 10
## Ansible playbook to provision Netscaler VIPs.
# Requires: nitrosdk-python
# 2018 (v.01) - Playbook from www.davideaves.com
---
- name: Netscaler VIP provision
  hosts: netscaler
  connection: local
  gather_facts: False

  vars:

    ansible_connection: "local"
    ansible_python_interpreter: "/usr/bin/env python"

    state: 'present'

    lbvip:
      name: testvip
      address: 203.0.113.1
      server:
        - name: 'server-1'
          address: '192.0.2.1'
          description: 'Ansible Test Server 1'
          disabled: 'true'
        - name: 'server-2'
          address: '192.0.2.2'
          description: 'Ansible Test Server 2'
          disabled: 'true'
        - name: 'server-3'
          address: '192.0.2.3'
          description: 'Ansible Test Server 3'
          disabled: 'true'
        - name: 'server-4'
          address: '192.0.2.4'
          description: 'Ansible Test Server 4'
          disabled: 'true'
        - name: 'server-5'
          address: '192.0.2.5'
          description: 'Ansible Test Server 5'
          disabled: 'true'
        - name: 'server-6'
          address: '192.0.2.6'
          description: 'Ansible Test Server 6'
          disabled: 'true'
        - name: 'server-7'
          address: '192.0.2.7'
          description: 'Ansible Test Server 7'
          disabled: 'true'
        - name: 'server-8'
          address: '192.0.2.8'
          description: 'Ansible Test Server 8'
          disabled: 'true'
      vserver:
        - port: '80'
          description: 'Generic service running on 80'
          type: 'HTTP'
          method: 'LEASTCONNECTION'
          persistence: 'SOURCEIP'
        - port: '443'
          description: 'Generic service running on 443'
          type: 'SSL_BRIDGE'
          method: 'LEASTCONNECTION'
          persistence: 'SOURCEIP'
        - port: '8080'
          description: 'Generic service running on 8080'
          type: 'HTTP'
          method: 'LEASTCONNECTION'
          persistence: 'SOURCEIP'
        - port: '8081'
          description: 'Generic service running on 8081'
          type: 'HTTP'
          method: 'LEASTCONNECTION'
          persistence: 'SOURCEIP'
        - port: '8443'
          description: 'Generic service running on 8443'
          type: 'SSL_BRIDGE'
          method: 'LEASTCONNECTION'
          persistence: 'SOURCEIP'

  tasks:

    - name: Build lbvip and all related componets.
      block:
      - local_action:
          module: netscaler_server
          nsip: "{{ inventory_hostname }}"
          nitro_user: "{{ nitro_user | default('nsroot') }}"
          nitro_pass: "{{ nitro_pass | default('nsroot') }}"
          nitro_protocol: "https"
          validate_certs: no
          state: "{{ state }}"
          name: "{{ item.name }}"
          ipaddress: "{{ item.address }}"
          comment: "{{ item.description | default('Ansible Created') }}"
          disabled: "{{ item.disabled | default('false') }}"
        with_items: "{{ lbvip.server }}"
      - local_action:
          module: netscaler_lb_monitor
          nsip: "{{ inventory_hostname }}"
          nitro_user: "{{ nitro_user | default('nsroot') }}"
          nitro_pass: "{{ nitro_pass | default('nsroot') }}"
          nitro_protocol: "https"
          validate_certs: no
          state: "{{ state }}"
          monitorname: "tcp_{{ lbvip.name }}_{{ item.port }}"
          type: TCP
          destport: "{{ item.port }}"
        with_items: "{{ lbvip.vserver }}"
        no_log: false
      - local_action:
          module: copy
          content: "{{ lookup('template', 'templates/netscaler_servicegroup.j2') }}"
          dest: "/tmp/svg_{{ lbvip.name }}_{{ item.port }}.yaml"
          mode: "0644"
        with_items: "{{ lbvip.vserver }}"
        changed_when: false
      - include_tasks: "/tmp/svg_{{ lbvip.name }}_{{ item.port }}.yaml"
        with_items: "{{ lbvip.vserver }}"
      - local_action:
          module: file
          state: absent
          path: "/tmp/svg_{{ lbvip.name }}_{{ item.port }}.yaml"
        with_items: "{{ lbvip.vserver }}"
        changed_when: false
      - local_action:
          module: netscaler_lb_vserver
          nsip: "{{ inventory_hostname }}"
          nitro_user: "{{ nitro_user | default('nsroot') }}"
          nitro_pass: "{{ nitro_pass | default('nsroot') }}"
          nitro_protocol: "https"
          validate_certs: no
          state: "{{ state }}"
          name: "vs_{{ lbvip.name }}_{{ item.port }}"
          servicetype: "{{ item.type }}"
          ipv46: "{{ lbvip.address }}"
          port: "{{ item.port }}"
          lbmethod: "{{ item.method | default('LEASTCONNECTION') }}"
          persistencetype: "{{ item.persistence | default('SOURCEIP') }}"
          servicegroupbindings:
            - servicegroupname: "svg_{{ lbvip.name }}_{{ item.port }}"
        with_items: "{{ lbvip.vserver }}"
      when: state == "present"

    - name: Destroy lbvip and all related componets.
      block:
      - local_action:
          module: netscaler_lb_vserver
          nsip: "{{ inventory_hostname }}"
          nitro_user: "{{ nitro_user | default('nsroot') }}"
          nitro_pass: "{{ nitro_pass | default('nsroot') }}"
          nitro_protocol: "https"
          validate_certs: no
          state: "{{ state }}"
          name: "vs_{{ lbvip.name }}_{{ item.port }}"
        with_items: "{{ lbvip.vserver }}"
      - local_action:
          module: netscaler_servicegroup
          nsip: "{{ inventory_hostname }}"
          nitro_user: "{{ nitro_user | default('nsroot') }}"
          nitro_pass: "{{ nitro_pass | default('nsroot') }}"
          nitro_protocol: "https"
          validate_certs: no
          state: "{{ state }}"
          servicegroupname: "svg_{{ lbvip.name }}_{{ item.port }}"
        with_items: "{{ lbvip.vserver }}"
      - local_action:
          module: netscaler_lb_monitor
          nsip: "{{ inventory_hostname }}"
          nitro_user: "{{ nitro_user | default('nsroot') }}"
          nitro_pass: "{{ nitro_pass | default('nsroot') }}"
          nitro_protocol: "https"
          validate_certs: no
          state: "{{ state }}"
          monitorname: "tcp_{{ lbvip.name }}_{{ item.port }}"
          type: TCP
        with_items: "{{ lbvip.vserver }}"
      - local_action:
          module: netscaler_server
          nsip: "{{ inventory_hostname }}"
          nitro_user: "{{ nitro_user | default('nsroot') }}"
          nitro_pass: "{{ nitro_pass | default('nsroot') }}"
          nitro_protocol: "https"
          validate_certs: no
          state: "{{ state }}"
          name: "{{ item.name }}"
        with_items: "{{ lbvip.server }}"
      when: state == "absent"

#!/usr/bin/ansible-playbook -f 10 ## Ansible playbook to provision Netscaler VIPs. # Requires: nitrosdk-python # 2018 (v.01) - Playbook from www.davideaves.com --- - name: Netscaler VIP provision hosts: netscaler connection: local gather_facts: False vars: ansible_connection: "local" ansible_python_interpreter: "/usr/bin/env python" state: 'present' lbvip: name: testvip address: 203.0.113.1 server: - name: 'server-1' address: '192.0.2.1' description: 'Ansible Test Server 1' disabled: 'true' - name: 'server-2' address: '192.0.2.2' description: 'Ansible Test Server 2' disabled: 'true' - name: 'server-3' address: '192.0.2.3' description: 'Ansible Test Server 3' disabled: 'true' - name: 'server-4' address: '192.0.2.4' description: 'Ansible Test Server 4' disabled: 'true' - name: 'server-5' address: '192.0.2.5' description: 'Ansible Test Server 5' disabled: 'true' - name: 'server-6' address: '192.0.2.6' description: 'Ansible Test Server 6' disabled: 'true' - name: 'server-7' address: '192.0.2.7' description: 'Ansible Test Server 7' disabled: 'true' - name: 'server-8' address: '192.0.2.8' description: 'Ansible Test Server 8' disabled: 'true' vserver: - port: '80' description: 'Generic service running on 80' type: 'HTTP' method: 'LEASTCONNECTION' persistence: 'SOURCEIP' - port: '443' description: 'Generic service running on 443' type: 'SSL_BRIDGE' method: 'LEASTCONNECTION' persistence: 'SOURCEIP' - port: '8080' description: 'Generic service running on 8080' type: 'HTTP' method: 'LEASTCONNECTION' persistence: 'SOURCEIP' - port: '8081' description: 'Generic service running on 8081' type: 'HTTP' method: 'LEASTCONNECTION' persistence: 'SOURCEIP' - port: '8443' description: 'Generic service running on 8443' type: 'SSL_BRIDGE' method: 'LEASTCONNECTION' persistence: 'SOURCEIP' tasks: - name: Build lbvip and all related componets. block: - local_action: module: netscaler_server nsip: "{{ inventory_hostname }}" nitro_user: "{{ nitro_user | default('nsroot') }}" nitro_pass: "{{ nitro_pass | default('nsroot') }}" nitro_protocol: "https" validate_certs: no state: "{{ state }}" name: "{{ item.name }}" ipaddress: "{{ item.address }}" comment: "{{ item.description | default('Ansible Created') }}" disabled: "{{ item.disabled | default('false') }}" with_items: "{{ lbvip.server }}" - local_action: module: netscaler_lb_monitor nsip: "{{ inventory_hostname }}" nitro_user: "{{ nitro_user | default('nsroot') }}" nitro_pass: "{{ nitro_pass | default('nsroot') }}" nitro_protocol: "https" validate_certs: no state: "{{ state }}" monitorname: "tcp_{{ lbvip.name }}_{{ item.port }}" type: TCP destport: "{{ item.port }}" with_items: "{{ lbvip.vserver }}" no_log: false - local_action: module: copy content: "{{ lookup('template', 'templates/netscaler_servicegroup.j2') }}" dest: "/tmp/svg_{{ lbvip.name }}_{{ item.port }}.yaml" mode: "0644" with_items: "{{ lbvip.vserver }}" changed_when: false - include_tasks: "/tmp/svg_{{ lbvip.name }}_{{ item.port }}.yaml" with_items: "{{ lbvip.vserver }}" - local_action: module: file state: absent path: "/tmp/svg_{{ lbvip.name }}_{{ item.port }}.yaml" with_items: "{{ lbvip.vserver }}" changed_when: false - local_action: module: netscaler_lb_vserver nsip: "{{ inventory_hostname }}" nitro_user: "{{ nitro_user | default('nsroot') }}" nitro_pass: "{{ nitro_pass | default('nsroot') }}" nitro_protocol: "https" validate_certs: no state: "{{ state }}" name: "vs_{{ lbvip.name }}_{{ item.port }}" servicetype: "{{ item.type }}" ipv46: "{{ lbvip.address }}" port: "{{ item.port }}" lbmethod: "{{ item.method | default('LEASTCONNECTION') }}" persistencetype: "{{ item.persistence | default('SOURCEIP') }}" servicegroupbindings: - servicegroupname: "svg_{{ lbvip.name }}_{{ item.port }}" with_items: "{{ lbvip.vserver }}" when: state == "present" - name: Destroy lbvip and all related componets. block: - local_action: module: netscaler_lb_vserver nsip: "{{ inventory_hostname }}" nitro_user: "{{ nitro_user | default('nsroot') }}" nitro_pass: "{{ nitro_pass | default('nsroot') }}" nitro_protocol: "https" validate_certs: no state: "{{ state }}" name: "vs_{{ lbvip.name }}_{{ item.port }}" with_items: "{{ lbvip.vserver }}" - local_action: module: netscaler_servicegroup nsip: "{{ inventory_hostname }}" nitro_user: "{{ nitro_user | default('nsroot') }}" nitro_pass: "{{ nitro_pass | default('nsroot') }}" nitro_protocol: "https" validate_certs: no state: "{{ state }}" servicegroupname: "svg_{{ lbvip.name }}_{{ item.port }}" with_items: "{{ lbvip.vserver }}" - local_action: module: netscaler_lb_monitor nsip: "{{ inventory_hostname }}" nitro_user: "{{ nitro_user | default('nsroot') }}" nitro_pass: "{{ nitro_pass | default('nsroot') }}" nitro_protocol: "https" validate_certs: no state: "{{ state }}" monitorname: "tcp_{{ lbvip.name }}_{{ item.port }}" type: TCP with_items: "{{ lbvip.vserver }}" - local_action: module: netscaler_server nsip: "{{ inventory_hostname }}" nitro_user: "{{ nitro_user | default('nsroot') }}" nitro_pass: "{{ nitro_pass | default('nsroot') }}" nitro_protocol: "https" validate_certs: no state: "{{ state }}" name: "{{ item.name }}" with_items: "{{ lbvip.server }}" when: state == "absent"

The following is the Jinja2 template that creates the netscaler_servicegroup task. An important thing to note is my use of the RAW block. When the task is created and stored in /tmp it does not contain any account credentials, instead I preserve the variable in the raw to prevent leaking sensitive information to anyone who may be snooping around on the server while the playbook is running.

templates/netscaler_servicegroup.j2: ASCII text, with CRLF line terminators

---
- local_action:
    module: netscaler_servicegroup
    nsip: {% raw %}"{{ inventory_hostname }}"
{% endraw %}
    nitro_user: {% raw %}"{{ nitro_user }}"
{% endraw %}
    nitro_pass: {% raw %}"{{ nitro_pass }}"
{% endraw %}
    nitro_protocol: "https"
    validate_certs: no

    state: "{{ state | default('present') }}"

    servicegroupname: "svg_{{ lbvip.name }}_{{ item.port }}"
    comment: "{{ item.description | default('Ansible Created') }}"
    servicetype: "{{ item.type }}"
    servicemembers:
{% for i in lbvip.server %}
      - servername: "{{ i.name }}"
        port: "{{ item.port }}"
{% endfor %}
    monitorbindings:
      - monitorname: "tcp_{{ lbvip.name }}_{{ item.port }}"

Using Ansible to perform a Netscaler backup

04. December 2018 · Comments Off · Categories: Ansible, Load Balancing, NetScaler · Tags: ansible, api, citrix, netscaler, systembackup

The following Ansible playbook is a rewrite of a script from a long time ago to perform backups of a Netscaler. As far as I know, there are no native Ansible or Vendor modules to perform a system backup. Within the playbook I am simply performing a raw call using the URI module against the Nitro API and fetching the backup file.

The following Vendor links contain good/related reference information:

Citrix Product Documentation: Backup and restore
Netscaler Development Documentation: systembackup
Netscaler Development Documentation: Direct NITRO API calls

netscaler_systembackup.yaml: a /usr/bin/ansible-playbook -f 10 script text executable, ASCII text

#!/usr/bin/ansible-playbook -f 10
## Ansible playbook to perform a full backup of Netscaler systems
## 2018 (v.01) - Playbook from www.davideaves.com
---
- name: Netscaler full backup
  hosts: netscalers
  connection: local
  gather_facts: False

  vars:

    ansible_connection: "local"
    ansible_python_interpreter: "/usr/bin/env python"

    backup_location: "/srv/nsbackup"

    ns_sys_backup: "/var/ns_sys_backup"

  tasks:

    - name: Check backup file status
      local_action:
        module: stat
        path: "{{ backup_location }}/{{ inventory_hostname }}_{{ lookup('pipe', 'date +%Y%m%d') }}_nsbackup.tgz"
      register: stat_result

    - name: Check backup directory location
      local_action:
        module: file
        path: "{{ backup_location }}"
        state: directory
        mode: 0775
        recurse: yes
      run_once: True
      when: stat_result.stat.exists == False

    - name: Full backup of Netscaler configuration.
      block:

      - name: Create Netscaler system backup
        local_action:
          module: uri
          url: "https://{{ inventory_hostname }}/nitro/v1/config/systembackup?action=create"
          method: POST
          validate_certs: no
          return_content: yes
          headers:
            X-NITRO-USER: "{{ nitro_user | default('nsroot') }}"
            X-NITRO-PASS: "{{ nitro_pass | default('nsroot') }}"
          body_format: json
          body: 
            systembackup:
              filename: "{{ inventory_hostname | hash('md5') }}"
              level: full
              comment: Ansible Generated Backup

      - name: Fetch Netscaler system backup
        local_action:
          module: uri
          url: "https://{{ inventory_hostname }}/nitro/v1/config/systemfile?args=filename:{{ inventory_hostname | hash('md5') }}.tgz,filelocation:{{ ns_sys_backup | replace('/','%2F') }}"
          method: GET
          status_code: 200
          validate_certs: no
          return_content: yes
          headers:
            X-NITRO-USER: "{{ nitro_user | default('nsroot') }}"
            X-NITRO-PASS: "{{ nitro_pass | default('nsroot') }}"
        register: result

      - name: Save Netscaler system backup to backup directory
        local_action: "shell echo '{{ result.json.systemfile[0].filecontent }}' | base64 -d > '{{ backup_location }}/{{ inventory_hostname }}_{{ lookup('pipe', 'date +%Y%m%d') }}_nsbackup.tgz'"

      - name: Chmod saved backup file permissions
        local_action:
          module: file
          path: "{{ backup_location }}/{{ inventory_hostname }}_{{ lookup('pipe', 'date +%Y%m%d') }}_nsbackup.tgz"
          mode: 0644

      always:

      - name: Delete system backup from Netscaler
        local_action:
          module: uri
          url: "https://{{ inventory_hostname }}/nitro/v1/config/systembackup/{{ inventory_hostname | hash('md5') }}.tgz"
          method: DELETE
          validate_certs: no
          return_content: yes
          headers:
            X-NITRO-USER: "{{ nitro_user | default('nsroot') }}"
            X-NITRO-PASS: "{{ nitro_pass | default('nsroot') }}"

      - name: Locate backup files older than 90 days
        local_action:
          module: find
          paths: "{{ backup_location }}"
          age: "1d"
        run_once: true
        register: files_matched

      - name: Purge old backup files
        local_action:
          module: file
          path: "{{ item.path }}"
          state: absent
        run_once: true
        with_items: "{{ files_matched.files }}"

      when: stat_result.stat.exists == False

#!/usr/bin/ansible-playbook -f 10 ## Ansible playbook to perform a full backup of Netscaler systems ## 2018 (v.01) - Playbook from www.davideaves.com --- - name: Netscaler full backup hosts: netscalers connection: local gather_facts: False vars: ansible_connection: "local" ansible_python_interpreter: "/usr/bin/env python" backup_location: "/srv/nsbackup" ns_sys_backup: "/var/ns_sys_backup" tasks: - name: Check backup file status local_action: module: stat path: "{{ backup_location }}/{{ inventory_hostname }}_{{ lookup('pipe', 'date +%Y%m%d') }}_nsbackup.tgz" register: stat_result - name: Check backup directory location local_action: module: file path: "{{ backup_location }}" state: directory mode: 0775 recurse: yes run_once: True when: stat_result.stat.exists == False - name: Full backup of Netscaler configuration. block: - name: Create Netscaler system backup local_action: module: uri url: "https://{{ inventory_hostname }}/nitro/v1/config/systembackup?action=create" method: POST validate_certs: no return_content: yes headers: X-NITRO-USER: "{{ nitro_user | default('nsroot') }}" X-NITRO-PASS: "{{ nitro_pass | default('nsroot') }}" body_format: json body: systembackup: filename: "{{ inventory_hostname | hash('md5') }}" level: full comment: Ansible Generated Backup - name: Fetch Netscaler system backup local_action: module: uri url: "https://{{ inventory_hostname }}/nitro/v1/config/systemfile?args=filename:{{ inventory_hostname | hash('md5') }}.tgz,filelocation:{{ ns_sys_backup | replace('/','%2F') }}" method: GET status_code: 200 validate_certs: no return_content: yes headers: X-NITRO-USER: "{{ nitro_user | default('nsroot') }}" X-NITRO-PASS: "{{ nitro_pass | default('nsroot') }}" register: result - name: Save Netscaler system backup to backup directory local_action: "shell echo '{{ result.json.systemfile[0].filecontent }}' | base64 -d > '{{ backup_location }}/{{ inventory_hostname }}_{{ lookup('pipe', 'date +%Y%m%d') }}_nsbackup.tgz'" - name: Chmod saved backup file permissions local_action: module: file path: "{{ backup_location }}/{{ inventory_hostname }}_{{ lookup('pipe', 'date +%Y%m%d') }}_nsbackup.tgz" mode: 0644 always: - name: Delete system backup from Netscaler local_action: module: uri url: "https://{{ inventory_hostname }}/nitro/v1/config/systembackup/{{ inventory_hostname | hash('md5') }}.tgz" method: DELETE validate_certs: no return_content: yes headers: X-NITRO-USER: "{{ nitro_user | default('nsroot') }}" X-NITRO-PASS: "{{ nitro_pass | default('nsroot') }}" - name: Locate backup files older than 90 days local_action: module: find paths: "{{ backup_location }}" age: "1d" run_once: true register: files_matched - name: Purge old backup files local_action: module: file path: "{{ item.path }}" state: absent run_once: true with_items: "{{ files_matched.files }}" when: stat_result.stat.exists == False

Convert ASA access-list rules to a parseable YAML format.

17. November 2018 · Comments Off · Categories: AWK, Cisco, Firewall, Linux Scripts, Networking · Tags: asa, awk, cisco, firewall, linux, yaml

This script spun out of a string of firewall migrations off the legacy ASA platform, I need the ability to convert access-lists to a parseable format. There are multiple reasons for needing this script. First is for human readability and auditing purposes. Second is to have a parseable rule base for duplication or migration to other firewall types.

ASA_acls.sh: Bourne-Again shell script text executable, ASCII text

#!/bin/bash
## Convert ASA access-list rules to a parseable YAML format.
## 2018 (v.01) - Script from www.davideaves.com
 
### VARIABLES ###
 
asa_config_file="${1}"
search_string="${2}"
 
### MAIN SCRIPT ###
 
[ -z "${asa_config_file}" ] && { echo -e "${0} - ERROR: missing ASA config"; exit 0; }
 
for ACCESSGROUP in `awk '/^access-group /{print $2}' "${asa_config_file}" | sort --ignore-case`
 do
 
  echo "${ACCESSGROUP}:"
  awk 'BEGIN{ REMARK=""; ACTION=""; SERVICE=""; SOURCE=""; DESTINATION=""; PORT=""; LOG=""; DISABLED=""; previous="" }
 
        # convert number to bits
        function bits(N){
          c = 0
          for(i=0; i<8; ++i) if(and(2**i, N)) ++c
          return c
        }
 
        # convert ipv4 to prefix
        function to_prefix(mask) {
          split(mask, octet, ".")
          return bits(octet[1]) + bits(octet[2]) + bits(octet[3]) + bits(octet[4])
        }
 
        # test if a string is an ipv4 address
        function is_v4(address) {
          split(address, octet, ".")
          if ( octet[1] <= 255 && octet[2] <= 255 && octet[3] <= 255 && octet[4] <= 255 )
          return address
        }
 
        # Only look at access-lists lines
        /^access-list '''${ACCESSGROUP}''' .*'''${search_string}'''/{
 
        # If line is a remark store it else continue
        if ( $3 == "remark" ) { $1=$2=$3=""; REMARK=substr($0,4) }
        else { $1=$2=$3=""; gsub("^   ", "")
 
          # Itterate through columns
          for(col = 1; col <= NF; col++) {
 
           # Append prefix to SOURCE & DESTINATION
           if ( is_v4(previous) && is_v4($col) ) {
            if ( DESTINATION != "" ) { DESTINATION=DESTINATION"/"to_prefix($col); previous="" }
            else if ( SOURCE != "" ) { SOURCE=SOURCE"/"to_prefix($col); previous="" }
          } else {
 
            # Determine col variable
            if ( col == "1" ) { ACTION=$col; SERVICE=""; SOURCE=""; DESTINATION=""; PORT=""; LOG=""; DISABLED=""; previous="" }
            else if ( $col ~ /^(eq|interface|object|object-group)$/ ) { previous=$col }
            else if ( SERVICE == "" && $col !~ /^(host|object|object-group)$/ ) { SERVICE=$col; PORT=""; previous="" }
            else if ( SOURCE == "" && $col !~ /^(host|object|object-group)$/ ) {
              if ( previous == "interface" ) { SOURCE=previous"/"$col }
              else { SOURCE=$col }; PORT=""; previous=to_prefix($col) }
            else if ( DESTINATION == "" && $col !~ /^(host|object|object-group)$/ ) {
              if ( previous == "interface" ) { DESTINATION=previous"/"$col }
              else { DESTINATION=$col }; PORT=""; previous=to_prefix($col) }
            else if ( previous ~ /^(eq|object-group)$/ ) { PORT=$col; previous="" }
            else if ( $col == "log" ) { LOG=$col; previous="" }
            else if ( $col == "inactive" ) { DISABLED=$col; previous="" }
            else { LAST=$col; previous="" }
 
          }
 
        }}
 
        # Display the output
        if ( DESTINATION != "" ) { count++
          print "  - name: '''${ACCESSGROUP}''' rule",count,"line",NR
          print "    debug:",$0
          if ( REMARK != "" ) { print "    description:",REMARK }
          print "    action:",ACTION
          print "    source:",SOURCE
          print "    destination:",DESTINATION
          if ( PORT == "" ) { print "    service:",SERVICE }
          else { print "    service:",SERVICE"/"PORT }
          if ( LOG != "" ) { print "    log: true" }
          if ( DISABLED != "" ) { print "    disabled: true" }
          REMARK=""; ACTION=""; SERVICE=""; SOURCE=""; DESTINATION=""; PORT=""; LOG=""; DISABLED=""; previous=""
        }
 
  }' "${asa_config_file}"
 
done

#!/bin/bash ## Convert ASA access-list rules to a parseable YAML format. ## 2018 (v.01) - Script from www.davideaves.com ### VARIABLES ### asa_config_file="${1}" search_string="${2}" ### MAIN SCRIPT ### [ -z "${asa_config_file}" ] && { echo -e "${0} - ERROR: missing ASA config"; exit 0; } for ACCESSGROUP in `awk '/^access-group /{print $2}' "${asa_config_file}" | sort --ignore-case` do echo "${ACCESSGROUP}:" awk 'BEGIN{ REMARK=""; ACTION=""; SERVICE=""; SOURCE=""; DESTINATION=""; PORT=""; LOG=""; DISABLED=""; previous="" } # convert number to bits function bits(N){ c = 0 for(i=0; i<8; ++i) if(and(2**i, N)) ++c return c } # convert ipv4 to prefix function to_prefix(mask) { split(mask, octet, ".") return bits(octet[1]) + bits(octet[2]) + bits(octet[3]) + bits(octet[4]) } # test if a string is an ipv4 address function is_v4(address) { split(address, octet, ".") if ( octet[1] <= 255 && octet[2] <= 255 && octet[3] <= 255 && octet[4] <= 255 ) return address } # Only look at access-lists lines /^access-list '''${ACCESSGROUP}''' .*'''${search_string}'''/{ # If line is a remark store it else continue if ( $3 == "remark" ) { $1=$2=$3=""; REMARK=substr($0,4) } else { $1=$2=$3=""; gsub("^ ", "") # Itterate through columns for(col = 1; col <= NF; col++) { # Append prefix to SOURCE & DESTINATION if ( is_v4(previous) && is_v4($col) ) { if ( DESTINATION != "" ) { DESTINATION=DESTINATION"/"to_prefix($col); previous="" } else if ( SOURCE != "" ) { SOURCE=SOURCE"/"to_prefix($col); previous="" } } else { # Determine col variable if ( col == "1" ) { ACTION=$col; SERVICE=""; SOURCE=""; DESTINATION=""; PORT=""; LOG=""; DISABLED=""; previous="" } else if ( $col ~ /^(eq|interface|object|object-group)$/ ) { previous=$col } else if ( SERVICE == "" && $col !~ /^(host|object|object-group)$/ ) { SERVICE=$col; PORT=""; previous="" } else if ( SOURCE == "" && $col !~ /^(host|object|object-group)$/ ) { if ( previous == "interface" ) { SOURCE=previous"/"$col } else { SOURCE=$col }; PORT=""; previous=to_prefix($col) } else if ( DESTINATION == "" && $col !~ /^(host|object|object-group)$/ ) { if ( previous == "interface" ) { DESTINATION=previous"/"$col } else { DESTINATION=$col }; PORT=""; previous=to_prefix($col) } else if ( previous ~ /^(eq|object-group)$/ ) { PORT=$col; previous="" } else if ( $col == "log" ) { LOG=$col; previous="" } else if ( $col == "inactive" ) { DISABLED=$col; previous="" } else { LAST=$col; previous="" } } }} # Display the output if ( DESTINATION != "" ) { count++ print " - name: '''${ACCESSGROUP}''' rule",count,"line",NR print " debug:",$0 if ( REMARK != "" ) { print " description:",REMARK } print " action:",ACTION print " source:",SOURCE print " destination:",DESTINATION if ( PORT == "" ) { print " service:",SERVICE } else { print " service:",SERVICE"/"PORT } if ( LOG != "" ) { print " log: true" } if ( DISABLED != "" ) { print " disabled: true" } REMARK=""; ACTION=""; SERVICE=""; SOURCE=""; DESTINATION=""; PORT=""; LOG=""; DISABLED=""; previous="" } }' "${asa_config_file}" done

Cisco Firepower Management Center (FMC) bulk modifications of policy rules.

10. August 2018 · Comments Off · Categories: Cisco, Firewall, Linux, Linux Scripts, Networking · Tags: api, cisco, curl, firepower, fmc, ftd, policy, script

As stated in my previous post, the Cisco Migration tools are very limited and Cisco has not shared any tools publicly that can perform bulk modifications. When migrating to a firepower unless all the previous access rules on the ASA had logging enabled any rule converted will continue to not have logging enabled. This is a major bummer for auditors, security teams or for anyone looking to eventually migrate L4 rules to L7 rules.

Just because I can, the following example script will bulk enable logBegin & sendEventsToFMC on all policy rules on a Firepower (FMC) appliance. This script is intended to be a reference, however modifying policy rules can broadly be broken down into the following steps:

GET the accessrule
Purge the metadata & link value trees from the json
Change, add or remove whatever values you want
PUT the accessrule

#!/bin/bash
## Example script that can bulk modify policy rules on a Cisco FMC.
## Requires: python:PyYAML,shyaml
## 2018 (v.01) - Script from www.davideaves.com
 
firepower="192.0.2.10"
username="apiuser"
password="api1234"
 
# Convert JSON to YAML
j2y() {
 python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' 2> /dev/null
}
 
# Convert YAML to JSON
y2j() {
 python -c 'import sys, yaml, json; y=yaml.load(sys.stdin.read()); print json.dumps(y)' 2> /dev/null
}
 
# Pop forbidden put values (metadata, links)
jpop() {
 python -c 'import sys, json; j=json.load(sys.stdin); j.pop("metadata"); j.pop("links"); print json.dumps(j)' 2> /dev/null
}
 
fmcauth() {
 ### Post credentials and eval header return data.
 
 if [[ -z "${auth_epoch}" || "${auth_epoch}" -lt "$(($(date +%s) - 1500))" ]]
  then
 
   if [ -z "${X_auth_access_toke}" ]
    then eval "auth_epoch=$(date +%s)"
        eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/generatetoken \
          -H "Authorization: Basic $(printf "${username}:${password}" | base64)" -D - |\
          awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')"
    else eval "auth_epoch=$(date +%s)"
         eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/refreshtoken \
          -H "X-auth-access-token: ${X_auth_access_token}" \
          -H "X-auth-refresh-token: ${X_auth_refresh_token}" -D - |\
          awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')"
   fi
 
 fi
}
 
# Get a list of all access control policies.
fmcauth && curl -skX GET https://${firepower}/api/fmc_config/v1/domain/${DOMAIN_UUID}/policy/accesspolicies \
 -H "X-auth-access-token: ${X_auth_access_token}" | j2y | shyaml get-value items | awk '/self:/{print $NF}' | while read POLICY
do fmcauth
 
   # Get a list of all access rules in each policy.
   curl -skX GET ${POLICY}/accessrules?limit=10000 -H "X-auth-access-token: ${X_auth_access_token}" |\
   j2y | shyaml get-value items 2> /dev/null | awk '/self:/{print $NF}' | while read RULE
   do
 
      # ID of the access rule.
      UUID="$(basename ${RULE})"
 
      # Collect the access rule resource and pop forbidden values.
      RESOURCE=`curl -skX GET ${RULE} -H "X-auth-access-token: ${X_auth_access_token}" | jpop`
 
      # Modify only if resource exists.
      if [[ ${RESOURCE} != *"Resource not found."* ]]
       then echo -ne "$(date) - PUT:${UUID} - MSG: "
 
       # Use sed to modify values in payload before putting back to FMC.
       curl -skX PUT ${RULE} \
        -H "Content-Type: application/json" \
        -H "X-auth-access-token: ${X_auth_access_token}" \
        -d "$(echo ${RESOURCE} | j2y | sed 's/logBegin: false/logBegin: true/;s/sendEventsToFMC: false/sendEventsToFMC: true/' | y2j)" && echo
      fi
 
   done
 
done

#!/bin/bash ## Example script that can bulk modify policy rules on a Cisco FMC. ## Requires: python:PyYAML,shyaml ## 2018 (v.01) - Script from www.davideaves.com firepower="192.0.2.10" username="apiuser" password="api1234" # Convert JSON to YAML j2y() { python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' 2> /dev/null } # Convert YAML to JSON y2j() { python -c 'import sys, yaml, json; y=yaml.load(sys.stdin.read()); print json.dumps(y)' 2> /dev/null } # Pop forbidden put values (metadata, links) jpop() { python -c 'import sys, json; j=json.load(sys.stdin); j.pop("metadata"); j.pop("links"); print json.dumps(j)' 2> /dev/null } fmcauth() { ### Post credentials and eval header return data. if [[ -z "${auth_epoch}" || "${auth_epoch}" -lt "$(($(date +%s) - 1500))" ]] then if [ -z "${X_auth_access_toke}" ] then eval "auth_epoch=$(date +%s)" eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/generatetoken \ -H "Authorization: Basic $(printf "${username}:${password}" | base64)" -D - |\ awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')" else eval "auth_epoch=$(date +%s)" eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/refreshtoken \ -H "X-auth-access-token: ${X_auth_access_token}" \ -H "X-auth-refresh-token: ${X_auth_refresh_token}" -D - |\ awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')" fi fi } # Get a list of all access control policies. fmcauth && curl -skX GET https://${firepower}/api/fmc_config/v1/domain/${DOMAIN_UUID}/policy/accesspolicies \ -H "X-auth-access-token: ${X_auth_access_token}" | j2y | shyaml get-value items | awk '/self:/{print $NF}' | while read POLICY do fmcauth # Get a list of all access rules in each policy. curl -skX GET ${POLICY}/accessrules?limit=10000 -H "X-auth-access-token: ${X_auth_access_token}" |\ j2y | shyaml get-value items 2> /dev/null | awk '/self:/{print $NF}' | while read RULE do # ID of the access rule. UUID="$(basename ${RULE})" # Collect the access rule resource and pop forbidden values. RESOURCE=`curl -skX GET ${RULE} -H "X-auth-access-token: ${X_auth_access_token}" | jpop` # Modify only if resource exists. if [[ ${RESOURCE} != *"Resource not found."* ]] then echo -ne "$(date) - PUT:${UUID} - MSG: " # Use sed to modify values in payload before putting back to FMC. curl -skX PUT ${RULE} \ -H "Content-Type: application/json" \ -H "X-auth-access-token: ${X_auth_access_token}" \ -d "$(echo ${RESOURCE} | j2y | sed 's/logBegin: false/logBegin: true/;s/sendEventsToFMC: false/sendEventsToFMC: true/' | y2j)" && echo fi done done

Cisco Firepower Management Center (FMC) bulk import & delete objects

08. August 2018 · Comments Off · Categories: Cisco, Firewall, Linux, Linux Scripts, Networking · Tags: cisco, curl, firepower, fmc, ftd, json, yaml

I’ve been doing a lot of migration work with the Cisco Firepower. I quickly noticed a general lack of tools to assist with migrations short of a few scattered and limited (hopefully incomplete and will continue to be developed) migration tools provided by Cisco:

enableMigrationTool.pl: Run as root on sacrificial FMC / will only convert ACL & NAT policies from an ASA config.
Cisco Firepower Migration Tool: Runs under Windows and assists with migrating only ACL & NAT policies from an ASA config.

If you are looking for tools to perform bulk rule changes or help convert from Layer4 rules to Layer7, like the PaloAlto Migration tool, you are out of luck. That being said, as an engineer trying to use the FMC, I quickly found the experience of working within the firepower interface slow, tedious and generally painful. This has forced me to try to interface with the API’s as much as possible to save time and to avoid using the interface.

First thing; API access needs to be enabled on the FMC; by default they are, but if disabled you can enable them by going to System > REST API Preferences and enabling them. If API’s are enabled the documentation will be accessible via: https://<FMCHOST>/api/api-explorer

API’s on the FMC take an imperative approach and are simple to trigger and depending on the HTTP method you can command the FMC to make changes. There are 4 basic methods the FMC will accept:

GET – Retrieves data from the specified object. GET is a read-only operation.
PUT – Adds supplied information to the specified object; returns a 404 Resource Not Found error if the object does not exist.
POST – Creates the object with the supplied information. POST operations are be followed with a payload consisting of JSON.
DELETE – Delete and object.

Concerning the types of objects that can be modified; the following are currently supported:

icmpv4objects
icmpv6objects
interfacegroups
networkgroups
networks
portobjectgroups
protocolportobjects
ranges
securityzones
slamonitors
urlgroups
urls
vlangrouptags
vlantags

As far as I can tell vpn policies, flexconfig and other objects of the sort must be created by hand. The following json & script are very rudimentary, however it is a working example that uses cURL to perform a bulk import of objects into a firepower. At the very least the curl commands can be used as a reference in your own projects.

fmc-objects.json: ASCII text, with very long lines

[{"value":"10.0.0.0","overridable":false,"name":"10.0.0.0_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.1","overridable":false,"name":"10.0.0.1_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.2","overridable":false,"name":"10.0.0.2_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.3","overridable":false,"name":"10.0.0.3_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.4","overridable":false,"name":"10.0.0.4_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.5","overridable":false,"name":"10.0.0.5_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.6","overridable":false,"name":"10.0.0.6_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.7","overridable":false,"name":"10.0.0.7_32","type":"Hosts","description":"Test REST API Object"},{"value":"10.0.0.8/29","overridable":false,"name":"10.0.0.8_29","type":"Networks","description":"Test REST API Object"}]

Because I do a lot with Ansible I have a strong preference to work with the YAML data serialization format. The script includes 2 unused functions that converts JSON to YAML and vice versa. I left those functions in this example script because they are handy and provide me flexibility of what and how I import. When posting to the FMC, its important to make sure the data is in JSON format and any metadata & link value trees have been popped. One thing I am lacking is a function to verify the object has been created, its easy to do by simply performing a GET after the POST, it just makes things slower and is simply not in this example.

fmc-objects.sh: Bourne-Again shell script, ASCII text executable

#!/bin/bash
## Example script that can bulk import and delete objects from a Cisco FMC.
## Requires: python:PyYAML,shyaml
## 2018 (v.01) - Script from www.davideaves.com
 
firepower="192.0.2.10"
username="apiuser"
password="api1234"
 
# Convert JSON to YAML
j2y() {
 python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' 2> /dev/null
}
 
# Convert YAML to JSON
y2j() {
 python -c 'import sys, yaml, json; y=yaml.load(sys.stdin.read()); print json.dumps(y)' 2> /dev/null
}
 
fmcauth() {
 ### Post credentials and eval header return data.
 
 if [[ -z "${auth_epoch}" || "${auth_epoch}" -lt "$(($(date +%s) - 1500))" ]]
  then
 
   if [ -z "${X_auth_access_toke}" ]
    then eval "auth_epoch=$(date +%s)"
        eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/generatetoken \
          -H "Authorization: Basic $(printf "${username}:${password}" | base64)" -D - |\
          awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')"
    else eval "auth_epoch=$(date +%s)"
         eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/refreshtoken \
          -H "X-auth-access-token: ${X_auth_access_token}" \
          -H "X-auth-refresh-token: ${X_auth_refresh_token}" -D - |\
          awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')"
   fi
 
 fi
}
 
 
### Bulk add diffrent type objects into FMC
 
# Cat single line example containing diffrent objects & use awk to extract json stanza.
cat "fmc-objects.json" | awk -F'[}{]' '{ for (i=1; i< =NF; i++) if(length($i) > 5) print "{"$i"}" }' |\
while read LINE
 do fmcauth
 
   # Determine Type & Value.
   TYPE="$(echo ${LINE} | shyaml get-value type | awk '{print tolower($0)}')"
   VALUE="$(echo ${LINE} | shyaml get-value value | awk '{print tolower($0)}')"
 
   echo -ne "$(date) - POST:${TYPE}>${VALUE} - MSG: "
 
   # Post new object to FMC.
   curl -skX POST https://${firepower}/api/fmc_config/v1/domain/${DOMAIN_UUID}/object/${TYPE} \
     -H "Content-Type: application/json" \
     -H "X-auth-access-token: ${X_auth_access_token}" \
     -d "${LINE}" && echo
done
 
### Bulk delete all unused objects from FMC...
 
# Valid objects types are: hosts icmpv4objects icmpv6objects interfacegroups networkgroups networks portobjectgroups protocolportobjects ranges securityzones slamonitors urlgroups urls vlangrouptags vlantags
for TYPE in hosts networks
 do fmcauth
    echo "### DELETEING ${TYPE} OBJECTS ###"
 
    # Get a list of objects.
    curl -skX GET https://${firepower}/api/fmc_config/v1/domain/${DOMAIN_UUID}/object/${TYPE}?limit=10000 \
     -H "X-auth-access-token: ${X_auth_access_token}" | j2y | shyaml get-value items 2> /dev/null | awk '/self:/{print $NF}' | while read SELF
    do  fmcauth
        echo -ne "$(date) - DELETE:${TYPE} - MSG: "
 
        # Delete the object.
        curl -skX DELETE ${SELF} -H "X-auth-access-token: ${X_auth_access_token}" && echo
    done
done

#!/bin/bash ## Example script that can bulk import and delete objects from a Cisco FMC. ## Requires: python:PyYAML,shyaml ## 2018 (v.01) - Script from www.davideaves.com firepower="192.0.2.10" username="apiuser" password="api1234" # Convert JSON to YAML j2y() { python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' 2> /dev/null } # Convert YAML to JSON y2j() { python -c 'import sys, yaml, json; y=yaml.load(sys.stdin.read()); print json.dumps(y)' 2> /dev/null } fmcauth() { ### Post credentials and eval header return data. if [[ -z "${auth_epoch}" || "${auth_epoch}" -lt "$(($(date +%s) - 1500))" ]] then if [ -z "${X_auth_access_toke}" ] then eval "auth_epoch=$(date +%s)" eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/generatetoken \ -H "Authorization: Basic $(printf "${username}:${password}" | base64)" -D - |\ awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')" else eval "auth_epoch=$(date +%s)" eval "$(curl -skX POST https://${firepower}/api/fmc_platform/v1/auth/refreshtoken \ -H "X-auth-access-token: ${X_auth_access_token}" \ -H "X-auth-refresh-token: ${X_auth_refresh_token}" -D - |\ awk '/(auth|DOMAIN|global)/{gsub(/[\r|:]/,""); gsub(/-/,"_",$1); print $1"=\""$2"\""}')" fi fi } ### Bulk add diffrent type objects into FMC # Cat single line example containing diffrent objects & use awk to extract json stanza. cat "fmc-objects.json" | awk -F'[}{]' '{ for (i=1; i< =NF; i++) if(length($i) > 5) print "{"$i"}" }' |\ while read LINE do fmcauth # Determine Type & Value. TYPE="$(echo ${LINE} | shyaml get-value type | awk '{print tolower($0)}')" VALUE="$(echo ${LINE} | shyaml get-value value | awk '{print tolower($0)}')" echo -ne "$(date) - POST:${TYPE}>${VALUE} - MSG: " # Post new object to FMC. curl -skX POST https://${firepower}/api/fmc_config/v1/domain/${DOMAIN_UUID}/object/${TYPE} \ -H "Content-Type: application/json" \ -H "X-auth-access-token: ${X_auth_access_token}" \ -d "${LINE}" && echo done ### Bulk delete all unused objects from FMC... # Valid objects types are: hosts icmpv4objects icmpv6objects interfacegroups networkgroups networks portobjectgroups protocolportobjects ranges securityzones slamonitors urlgroups urls vlangrouptags vlantags for TYPE in hosts networks do fmcauth echo "### DELETEING ${TYPE} OBJECTS ###" # Get a list of objects. curl -skX GET https://${firepower}/api/fmc_config/v1/domain/${DOMAIN_UUID}/object/${TYPE}?limit=10000 \ -H "X-auth-access-token: ${X_auth_access_token}" | j2y | shyaml get-value items 2> /dev/null | awk '/self:/{print $NF}' | while read SELF do fmcauth echo -ne "$(date) - DELETE:${TYPE} - MSG: " # Delete the object. curl -skX DELETE ${SELF} -H "X-auth-access-token: ${X_auth_access_token}" && echo done done