There are no built-in checks for this in OpenLayers, but it should be possible to implement one quite easily.
The Polygon class has a Components property which holds several LinearRings, the first ring beeing the outer ring and the consecutive ones (if any) represents holes.
To find self-intersections you could make a function that loops the induvidual points of each ring, forming linestrings and checking intersections.
I think an approach like this should work (this only checks if the outer ring is self-intersecting, but it should be quite easy to extend it to holes, just build segments from the holes and push to the segments array)
var format = new OpenLayers.Format.WKT();
var f1=format.read("POLYGON(1 1, 4 1, 4 4, 1 4, 2 5, 1 1)");
var f2=format.read("POLYGON(1 1, 4 1, 4 4, 1 4, 1 1)");
function checkSelfIntersection(polygon){
console.log(polygon);
if(polygon.CLASS_NAME=="OpenLayers.Geometry.Polygon"){
//checking only outer ring
var outer = polygon.components[0].components;
var segments = [];
for(var i=1;i<outer.length;i++){
var segment= new OpenLayers.Geometry.LineString([outer[i-1].clone(),outer [i].clone()]);
segments.push(segment);
}
for(var j=0;j<segments.length;j++){
if(segmentIntersects(segments[j],segments)){
return true;
}
}
}
return false;
}
function segmentIntersects(segment,segments){
for(var i=0;i<segments.length;i++){
if(!segments[i].equals(segment)){
if(segments[i].intersects(segment) && !startOrStopEquals(segments[i],segment)){
return true;
}
}
}
return false;
}
function startOrStopEquals(segment1,segment2){
if(segment1.components[0].equals(segment2.components[0])){
return true;
}
if(segment1.components[0].equals(segment2.components[1])){
return true;
}
if(segment1.components[1].equals(segment2.components[0])){
return true;
}
if(segment1.components[1].equals(segment2.components[1])){
return true;
}
return false;
}
console.log(checkSelfIntersection(f1.geometry));
console.log(checkSelfIntersection(f2.geometry));