Comments (9)
For anyone else who hits this... here's a quick script to clean things up. Designed to work from the CLI vs. lambda. Note, designed to work with waf not waf-regional (eg. alb vs. cloudfront wafs)
import boto3
import optparse
parser = optparse.OptionParser()
parser.add_option('--setid', help='SetID via aws waf list-set-ids or aws waf-regional list-set-ids')
parser.add_option('--profile', help='AWS Profile as in ~/.aws/credentials file')
(options, args) = parser.parse_args()
if not options.setid:
parser.error("Missing setid")
if not options.profile:
parser.error("Missing profile")
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
session = boto3.session.Session(profile_name= options.profile)
client = session.client("waf")
ipset = client.get_ip_set(IPSetId = options.setid)
count = 1
for chunk in chunks(ipset['IPSet']['IPSetDescriptors'], 500):
delete_set = []
for item in chunk:
delete_set.append({'Action':'DELETE','IPSetDescriptor':{'Type':item['Type'], 'Value':item['Value']}})
token = client.get_change_token()
response = client.update_ip_set( ChangeToken=token['ChangeToken'], IPSetId=options.setid, Updates=delete_set)
print "On count {0} of delete... ".format(count)
count += 1
from aws-waf-security-automations.
I ran into this problem today with v2.3.0. Deleting the stack would fail to delete the regional IP Reputation Lists Set.
Here's how I fixed it:
- Load console for WAF > IP Addresses condition > select my region > select "IP Reputation Lists Set" and saw 2,479 IP descriptors
- Show 1,000 descriptors per page
- Select all and delete the descriptors.
After I did this, I was able to delete the CloudFormation stack without a rate limit error. AFAICT, this is a manual attempt at what @alfaro28 suggested above
from aws-waf-security-automations.
Yes I came across this issue also. I modified one of the scripts to help me delete them quickly. This cleanup script could be run from a Lambda function and supplying it with the id's of the IPSets you want to delete. I have been doing a lot of CloudFormation testing so deleting all these IPSets is a pain in the AWS Console if you are using sets that contain 10000 ip addreses!
{ "ipSetIds": [ "402969fd-fba8-4dbb-9fcb-925ee51fa738", "a6ffd867-7c6d-4b93-b9e6-41e02cfeba52" ] }
The script I used is
var readline = require('readline');
var aws = require('aws-sdk');
var https = require('https');
var async = require('async');
// configure API retries
aws.config.update({
//region:'eu-west-1',
maxRetries: 3,
retryDelayOptions: {
base: 1000
}
});
//var waf = new aws.WAFRegional();
var waf = new aws.WAF();
var cloudwatch = new aws.CloudWatch();
var cloudformation = new aws.CloudFormation();
/**
- Maximum number of IP descriptors per IP Set
*/
var maxDescriptorsPerIpSet = 10000;
/**
- Maximum number of IP descriptors updates per call
*/
var maxDescriptorsPerIpSetUpdate = 1000;
/**
- Convert a dotted-decimal formated address to an integer
*/
function dottedToNumber(dotted) {
var splitted = dotted.split('.');
return (((((Number(splitted[0]) * 256) + Number(splitted[1])) * 256) + Number(splitted[2])) * 256) + Number(splitted[3]);
}
/**
- Convert an IPv4 address integer to dotted-decimal format
*/
function numberToDotted(number) {
var dotted = String(number % 256);
for (var j = 3; j > 0; j--) {
number = Math.floor(number / 256);
dotted = String(number % 256) + '.' + dotted;
}
return dotted;
}
/**
- Constructs a new object representing an IPv4 address range
- @Class
- @classdesc An IPv4 address range
- @param {List} list - The List object that the range is defined in
- @param {string|number} address - Either a number, a dotted decimal address, or a CIDR
- @param {number} [mask] - The mask, ignored if address is CIDR
/
function Range(list, address, mask) {
this.list = list;
// check to see if the address is in dotted-decimal format, optionally including the mask
if ((typeof address == 'string') && (address.indexOf('.') !== -1)) {
var slashPosition = address.indexOf('/');
if (slashPosition === -1) {
this.dotted = address;
this.mask = 32;
} else {
this.dotted = address.substring(0, slashPosition);
this.mask = Number(address.substring(slashPosition + 1));
}
this.number = dottedToNumber(this.dotted);
}
else {
this.number = Number(address);
this.mask = mask || 32;
this.dotted = numberToDotted(this.number);
}
this.cidr = this.dotted + '/' + this.mask;
this.lastNumber = this.number + Math.pow(2, 32 - this.mask);
}
/* - Test if the other range is contained within this one
- @param {Range} other - The other range
*/
Range.prototype.contains = function (other) {
return ((this.number <= other.number) && (this.lastNumber >= other.lastNumber));
};
Range.prototype.toString = function () {
return this.cidr;
};
/**
- Constructs a new object containing an URL to a reputation list
- @Class
- @classdesc An IP Reputation List
- @param {string} url - URL to the reputation list
- @param {string} prefix - Regular Expression prefix before the IP address
/
function List(url, prefix) {
this.url = url;
this.prefix = prefix || '';
// a regular expression to find the address or range on each line of the list, with an option prefix before it
this.regex = new RegExp('^' + this.prefix + '((?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])(?:/(?:3[0-2]|[1-2][0-9]|[0-9]))?)');
}
/* - Get ranges defined in list
- @param {function} callback - The callback function on completion
*/
List.prototype.getRanges = function (callback) {
var list = this;
var ranges = [];
https.get(this.url, function (response) {
// create a reader object to read the list one line at a time
var reader = readline.createInterface({ terminal: false, input: response });
reader.on('line', function (line) {
var result = list.regex.exec(line);
// if there is a result, a range has been found and a new range is created
if (result) {
ranges.push(new Range(list, result[1]));
}
});
reader.on('close', function () {
console.log(ranges.length + ' address ranges read from ' + list.url);
callback(null, ranges);
});
}).on('error', function (err) {
console.error('Error downloading ' + this.url, err);
callback(err);
});
};
List.prototype.equals = function (other) {
return this.url === other.url;
};
List.prototype.toString = function () {
return this.url;
};
/**
- Logs an array of ranges, with optional message, to console
- @param {Range[]} ranges - List of ranges
- @param {string} [message] - Message
- @param {number} [indent=0] - Number of tabs to indent text with
*/
function logRanges(ranges, message) {
if (message) {
console.log(ranges.length + ' ranges ' + message);
}
}
/**
- Sorts an array of ranges by largest first
- @param {Range[]} ranges - List of ranges
*/
function prioritizeRanges(ranges) {
ranges.sort(function (a, b) {
return a.mask - b.mask;
});
logRanges(ranges, 'after prioritzing');
}
/**
- Removes ranges from a list if they are contained within other ranges
- @param {Range[]} ranges - List of ranges
*/
function removeContainedRanges(ranges) {
for (var i = 0; i < ranges.length; i++) {
var range = ranges[i];
for (var j = 0; j < ranges.length; j++) {
var other = ranges[j];
if (range.contains(other) && (j !== i)) {
ranges.splice(j, 1);
if (j < i) {
i--;
}
j--;
}
}
}
logRanges(ranges, 'after removing contained ones');
}
/**
- Combine ranges into larger /8, /16, or /24 ranges
- @param {Range[]} ranges - Array of ranges
*/
function CombineRanges(ranges) {
// TODO: should check if we can combine ranges into a larger /8, /26, /24 ranges
}
/**
- Split ranges into smaller /8, /16, /24 or /32 ranges
- @param {Range[]} ranges - Array of ranges
*/
function splitRanges(ranges) {
// AWS WAF only support ranges with /8, /16, /24 or /32 masks
// Therefore, split ranges into ones that have the above masks
// For example = /15 can be decomposed into 2 /16 ranges, /17 can be decomposed into 64 /14 ranges
for (var i = 0; i < ranges.length; i++) {
var range = ranges[i];
var list = range.list;
var mask = range.mask;
var supportedMask = (mask <= 8 ? 8 : mask <= 16 ? 16 : mask <= 24 ? 24 : 32);
var supportedMaskDifference = supportedMask - mask;
// Check if the mask is not a /8, /16, /24 or /32
if (supportedMaskDifference > 0) {
var size = Math.pow(2, 32 - supportedMask);
var count = Math.pow(2, supportedMaskDifference);
var newRanges = [];
// create new ranges that have /8, /16, /24 or /32 masks to replace this
for (var j = 0; j < count; j++) {
newRanges.push(new Range(list, range.number + (j * size), supportedMask));
}
// Insert the new ranges into the array, removing this one
Array.prototype.splice.apply(ranges, [i, 1].concat(newRanges));
// move the pointer to after the newly-inserted ranges
i += newRanges.length - 1;
}
}
logRanges(ranges, 'after splitting to /8, /16, /24 or /32 ranges...');
}
/**
- Flattens an array of arrays into an arry
- @param {array[]} arr - Array of arrays
*/
function flattenArrayArray(arr) {
return arr.reduce(function (a, b) {
return a.concat(b);
}, []);
}
/**
- Flattens an array of objects into an array
- @param {array[]} arr - Array of objects
- @param {string} propertyName - Name of property of array elements to extract
*/
function flattenObjectArray(array, propertyName) {
return array.map(function (o) {
return o[propertyName];
});
}
/**
- Call context.done, loggin message to console
- @param {Context} context - Lambda context object
- @param {Error} err - Error object
- @param {String} message - Message
*/
function done(context, err, message) {
console[err ? 'error' : 'log'](message, err);
context.done(err, message);
}
function updateIPSets(updates){
async.waterfall([
function (callback) {
waf.getChangeToken({}, callback);
},
function (response, callback) {
console.log('Updating IP set ' + ipSetName + ' with ' + updates.length + ' updates');
waf.updateIPSet({
ChangeToken: response.ChangeToken,
IPSetId: ipSet.IPSetId,
Updates: updates
}, callback);
}
], function (err, response) {
if (err) {
console.error('Error updating IP set ' + ipSetName, err);
} else {
console.log('Updated IP set ' + ipSetName);
}
callback(err);
});
}
var IPSetDescriptor = function(type, value) {
this.Action = "DELETE";
this.Type = type;
this.Value = value;
};
/**
-
Main handler
*/
exports.handler = function (event, context) {
console.log('event: ' + JSON.stringify(event));
if (!event || !event.ipSetIds || (event.ipSetIds.length === 0)) {
done(context, null, 'Nothing to do');
} else {
async.parallel([
// get each waf ip set
function (callback) {
async.map(event.ipSetIds, function (IPSetId, callback) {
waf.getIPSet({ IPSetId: IPSetId }, callback);
}, function (err, ipSets) {
if (err) {
console.error('Error getting IP sets', err);
} else {
// ipSets is an array of objects with an IPSet property, so 'flatten' it
ipSets = flattenObjectArray(ipSets, 'IPSet');
console.log(ipSets.length + ' IP Sets in total');
}
callback(err, ipSets);
});
}
], function (err, rangesAndIPSets) {
if (err) {
done(context, err, 'Error getting ranges and/or IP sets');
} else {
// rangesAndIPSets is an array with two elements - the first is an array of ranges, the second an array of IPSets
var ranges = rangesAndIPSets[0];
var ipSets = rangesAndIPSets[0];
var tasks = [];
ipSets.forEach(function (ipSet, index) {
var ipSetName = ipSet.Name;
var ipSetDescriptors = ipSet.IPSetDescriptors;
var begin = index * maxDescriptorsPerIpSet;
var rangeSlice = ranges.slice(begin, begin + maxDescriptorsPerIpSet);
console.log('IP Set ' + ipSetName + ' has ' + ipSetDescriptors.length + ' descriptors and should have ' + rangeSlice.length);
var updates = [];
ipSetDescriptors.forEach(function (ipSetDescriptor) {
var cidr = ipSetDescriptor.Value;
var found;
// try to find the IPSet descriptor on the ranges slice
if (cidr == null){
// your code here.
}
if(cidr != null){
for (var i = 0; i < rangeSlice.length; i++) {
if (rangeSlice[i].cidr === cidr) {
rangeSlice.splice(i, 1);
found = true;
break;
}
}
// if this descriptor is not found in the ranges slice, it is deleted
if (!found) updates.push({ Action: 'DELETE', IPSetDescriptor: ipSetDescriptor });
}}); var updatesLength = updates.length; if (updatesLength > 0) { console.log('IP Set ' + ipSetName + ' requires ' + updatesLength + ' updates'); //console.log('IP Set ' + ipSetName + ' updates: ' + updates.map(function (o) { // return o.Action + ' ' + o.IPSetDescriptor.Value; //}).join(', ')); // limit the number of updates in a single call var batches = []; while (updates.length) { batches.push(updates.splice(0, maxDescriptorsPerIpSetUpdate)); } Array.prototype.push.apply(tasks, batches.map(function(updateBatch) { return function (callback) { async.waterfall([ function (callback) { waf.getChangeToken({}, callback); }, function (response, callback) { console.log('Updating IP set ' + ipSetName + ' with ' + updateBatch.length + ' updates'); waf.updateIPSet({ ChangeToken: response.ChangeToken, IPSetId: ipSet.IPSetId, Updates: updateBatch }, callback); } ], function (err, response) { if (err) { console.error('Error updating IP set ' + ipSetName, err); } else { console.log('Updated IP set ' + ipSetName); } callback(err); }); }; })); } else { // there are no updates for this IP Set console.log('No update required for IP set' + ipSetName); } }); if (tasks.length > 0) { // there are update tasks to be performed - i.e. there are IP Sets that require updating async.series(tasks, function (err) { var notFitCount = ranges.length - (ipSets.length * maxDescriptorsPerIpSet); done(context, err, err ? 'Error updating IP sets' : 'Updated IP sets' + (notFitCount > 0 ? ', ' + notFitCount + ' ranges unable to fit in IP sets' : '')); }); } else { done(context, null, 'No updates required for IP sets'); } } });
}
};
from aws-waf-security-automations.
Thanks for the code update systemsymbiosis. Will incorporate this fix in the next release!
from aws-waf-security-automations.
@hvital It seems you never did. We're still waiting.
from aws-waf-security-automations.
What I have ended up doing is using the above code as a Lambda function that I run using the IDs of the IP Rules created in the WAF as inputs. It is a bit of a pain getting the ID's and I could probably automate better. But every so often I use the function to clean up the account for any stray rules that have not been deleted after I have torn down my CloudFormation stack.
from aws-waf-security-automations.
Modified version of @jasonmcintosh code to work with waf-regional
import boto3
import optparse
parser = optparse.OptionParser()
parser.add_option('--setid', help='SetID via aws waf list-set-ids or aws waf-regional list-set-ids')
parser.add_option('--profile', help='AWS Profile as in ~/.aws/credentials file')
parser.add_option('--region', help='for example us-east-1')
(options, args) = parser.parse_args()
if not options.setid:
parser.error("Missing setid")
if not options.profile:
parser.error("Missing profile")
if not options.region:
parser.error("Missing region")
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
session = boto3.session.Session(profile_name= options.profile)
client = session.client("waf-regional", region_name=options.region)
ipset = client.get_ip_set(IPSetId = options.setid)
count = 1
for chunk in chunks(ipset['IPSet']['IPSetDescriptors'], 500):
delete_set = []
for item in chunk:
delete_set.append({'Action':'DELETE','IPSetDescriptor':{'Type':item['Type'], 'Value':item['Value']}})
token = client.get_change_token()
response = client.update_ip_set( ChangeToken=token['ChangeToken'], IPSetId=options.setid, Updates=delete_set)
print("On count {0} of delete... ".format(count))
count += 1
from aws-waf-security-automations.
Fixed in v2.1
from aws-waf-security-automations.
This issue occurred to me today when launching the stack documented here.
After the stack was created using the CloudFront endpoint type, I attempted to delete the stack and faced the issue above.
@jasonmcintosh script did the trick for me.
from aws-waf-security-automations.
Related Issues (20)
- Updating the stack removes configured IP set HOT 2
- Can we add the rule to detect JSON-Based SQL to prevent AWS WAF bypass HOT 1
- Fails to create AccessLoggingBucket in us-east-2 HOT 3
- Include least-privileged-access IAM role for deploying the stack HOT 1
- Incompatible request with urllib3>2 HOT 2
- Block hidden files and folders like .env and .git HOT 1
- WAF denying a request based on some custom condition HOT 1
- while build i get No module named 'lib.cw_metrics_util' HOT 4
- missing file HOT 1
- Runtime.ImportModuleError: Unable to import module 'helper': No module named 'lib.s3_util' HOT 2
- Need help writing Custom ACL HOT 3
- Most if not all of Lambda functions created by version 4.0.0 contain vulnerability in requests package of Python HOT 1
- Honeypot doesn't detect correct IP address with CloudFront and recommended cache configuration. HOT 4
- 4.0.2 helper.zip not available in cn-north-1 HOT 1
- Failing to create security-automations-for-aws-waf in il-central-1 HOT 3
- Support for Cloudfront realtime logs HOT 2
- Failing to create security-automations-for-aws-waf in il-central-1 HOT 3
- WAFWebACL Drift for Security Automations for AWS WAF Solution HOT 1
- Glue Table for WAF Access Logs is missing some of the log fields (e.g. labels) HOT 4
- allow cloudwatch logs destination for traffic logs, currently only supported target is s3 HOT 8
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from aws-waf-security-automations.